Release 4.2.0

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 include:
-  - project: 'externalci/ci-image'
+  - project: '$CI_TEMPLATE'
     ref: master
     file: '.gitlab-ci-default.yaml'
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "polyswarm_api"
-version = "4.1.0"
+version = "4.2.0"
 description = "Client library to simplify interacting with the PolySwarm consumer API"
 readme = "README.md"
 requires-python = ">=3.10,<4"
@@ -55,7 +55,7 @@ package-dir = { "" = "src" }
 where = ["src"]
 
 [tool.bumpversion]
-current_version = "4.1.0"
+current_version = "4.2.0"
 commit = true
 tag = false
 sign_tags = true

specs/01-architecture.md

@@ -270,7 +270,7 @@ Every HTTP-level error maps to a subclass of `PolyswarmException`:
 - 404 → `NotFoundException`
 - 422 → `FailedInstanceException`
 - 429 → `UsageLimitsExceededException`
-- 204 + JSON parser expected → `NoResultsException`
+- 204 on a request that expects data (JSON-parser GET **or** streaming download) → `NoResultsException` — the server did the work but matched nothing. **HEAD is exempt**: it returns the raw status code as the result (so `exists()` reads a 204 as "known-absent" rather than raising).
 - Other non-2xx → `RequestException`
 - Client-side validation failures (bad hash, missing kwarg) → `InvalidValueException`
 - Polling timeouts → `TimeoutException`

specs/02-resources.md

@@ -288,7 +288,7 @@ Wraps a single scan instance. Carries `id`, `sha256`, `upload_url`, the assertio
 **`known_good` / `known_good_sources`.** When the server flags this sha256 as a
 known-good binary, the response carries a `known_good` array — one
 `{tool, tool_metadata, created, updated}` entry per flagging feed (`nsrl`,
-`winbindex`, `winget`). `ArtifactInstance.known_good` is that raw list (or `None`
+`microsoft`, `commercial`). `ArtifactInstance.known_good` is that raw list (or `None`
 for a normal artifact / a server too old to emit the field — parsed with `.get()`,
 so older recorded responses parse to `None` with no behaviour change), and
 `known_good_sources` is the sorted, de-duplicated list of feed names derived from

specs/03-endpoints.md

@@ -25,7 +25,7 @@ The full catalogue of methods on the public client surface and which transport h
 
 | Method | Resource builder | Notes |
 |---|---|---|
-| `exists(hash_, hash_type=None, require_scan=False)` | `ArtifactInstance.exists_hash` | HEAD; returns `bool` from status code. |
+| `exists(hash_, hash_type=None, require_scan=False)` | `ArtifactInstance.exists_hash` | HEAD; `bool` from status code — `True` **only** for `200` (present). `204` means "absent" (the request succeeded but matched no artifact) and `404` also maps to absent, so both are `False`. Do **not** treat this as a generic `2xx` check: `204` is a successful status that means the opposite of "exists". |
 | `lookup(scan)` | `ArtifactInstance.lookup_uuid` | |
 | `rescan(hash_, hash_type=None, scan_config=None)` | `ArtifactInstance.rescan` | |
 | `rescan_id(scan, scan_config=None)` | `ArtifactInstance.rescan_id` | |

specs/99-open-questions.md

@@ -114,6 +114,7 @@ The 4.0 transport originally buffered (the adapter read `response.content` whole
 - detects a streaming parser (`request.result_parser is not None and not issubclass(request.result_parser, BaseJsonResource)`) and routes to `_execute_download`;
 - opens the response with `self._client.send(req, stream=True)` (status/headers available, body not read) — which also covers the auth-stripped off-domain S3 case (`download_archive`), since header suppression already goes through `build_request` + pop;
 - maps non-2xx via the shared `core._raise_for_status` (identical typed exceptions to the JSON path) after `aread()`-ing the small error body;
+- treats a **204 as "no matching artifact"** (the request succeeded but returned nothing) and raises `NoResultsException` — mirroring the shared `parse_response` 204 rule that the streaming path otherwise bypasses, so an absent download surfaces "no results" instead of silently writing a successful empty file (regression-guarded by `test_async_download_204_raises_no_results`);
 - has the parser class resolve a destination handle (`LocalArtifact.open_destination`), streams the body in chunk by chunk (`response.aiter_bytes(DOWNLOAD_CHUNK_SIZE)`), wraps the written handle (`LocalArtifact.from_written`), and removes a partially-written file it created;
 - closes the response in a `finally` (`aclose`).
 

src/polyswarm_api/__init__.py

@@ -1,5 +1,5 @@
 # https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names
-__version__ = '4.1.0'
+__version__ = '4.2.0'
 __release_url__ = 'https://api.github.com/repos/polyswarm/polyswarm-api/releases/latest'
 
 from . import api

src/polyswarm_api/aio/api.py

@@ -1025,7 +1025,8 @@ async def llm_report_create(self, instance_id=None, cape_sandbox_task_id=None, t
         report_task = await self._single(resources.ReportLLMPostProcessing.create(self,
                                                                instance_id=instance_id,
                                                                cape_sandbox_task_id=cape_sandbox_task_id,
-                                                               triage_sandbox_task_id=triage_sandbox_task_id))
+                                                               triage_sandbox_task_id=triage_sandbox_task_id,
+                                                               community=self.community))
         return report_task
 
     async def llm_report_get(self, report_task_id):
@@ -1651,8 +1652,11 @@ async def exists(self, hash_, hash_type=None, require_scan=False):
                 self, hash_.hash, hash_.hash_type, require_scan=require_scan,
             ),
         )
-        # exists_hash is a HEAD; ``result`` is the status code. Any 2xx means the
-        # artifact is known — the endpoint returns 200 for present / 404 for absent,
-        # so a 2xx check is correct and not brittle to a non-200 success code.
-        return int(result) // 100 == 2
+        # exists_hash is a HEAD; ``result`` is the status code. The endpoint returns
+        # 200 when the artifact is present and 204 when it is absent ("the request
+        # worked, but there was no matching artifact, so nothing was returned"). Only
+        # a 200 means the artifact exists — 204 (and 404) both mean "absent" → False.
+        # NB: this is deliberately ``== 200``, not a ``// 100 == 2`` 2xx check, because
+        # 204 is a successful 2xx that means the opposite of "exists".
+        return int(result) == 200
 

src/polyswarm_api/aio/session.py

@@ -148,6 +148,19 @@ async def _execute_download(self, request):
                 await response.aread()
                 _raise_for_status(response, request)
 
+            if response.status_code == 204:
+                # A 204 is the "no data / absent" signal on a download: the request
+                # succeeded but there is nothing to write ("the request worked, but
+                # there was no matching artifact, so nothing was returned"). The
+                # shared ``parse_response`` maps a parser-backed 204 to
+                # ``NoResultsException``; the streaming path bypasses ``parse_response``,
+                # so mirror that rule explicitly here. Without it, an absent artifact
+                # would be written out as a successful *empty* file instead of
+                # surfacing "no results".
+                raise exceptions.NoResultsException(
+                    request, 'The request returned no results.',
+                )
+
             pk = request.parser_kwargs or {}
             handle, name, created = request.result_parser.open_destination(
                 pk.get('folder'), pk.get('handle'), pk.get('artifact_name'), response,

src/polyswarm_api/api.py

@@ -1262,6 +1262,7 @@ def llm_report_create(
                 instance_id=instance_id,
                 cape_sandbox_task_id=cape_sandbox_task_id,
                 triage_sandbox_task_id=triage_sandbox_task_id,
+                community=self.community,
             )
         )
         return report_task
@@ -2001,7 +2002,10 @@ def exists(self, hash_, hash_type=None, require_scan=False):
                 require_scan=require_scan,
             ),
         )
-        # exists_hash is a HEAD; ``result`` is the status code. Any 2xx means the
-        # artifact is known — the endpoint returns 200 for present / 404 for absent,
-        # so a 2xx check is correct and not brittle to a non-200 success code.
-        return int(result) // 100 == 2
+        # exists_hash is a HEAD; ``result`` is the status code. The endpoint returns
+        # 200 when the artifact is present and 204 when it is absent ("the request
+        # worked, but there was no matching artifact, so nothing was returned"). Only
+        # a 200 means the artifact exists — 204 (and 404) both mean "absent" → False.
+        # NB: this is deliberately ``== 200``, not a ``// 100 == 2`` 2xx check, because
+        # 204 is a successful 2xx that means the opposite of "exists".
+        return int(result) == 200

src/polyswarm_api/session.py

@@ -151,6 +151,20 @@ def _execute_download(self, request):
                 response.read()
                 _raise_for_status(response, request)
 
+            if response.status_code == 204:
+                # A 204 is the "no data / absent" signal on a download: the request
+                # succeeded but there is nothing to write ("the request worked, but
+                # there was no matching artifact, so nothing was returned"). The
+                # shared ``parse_response`` maps a parser-backed 204 to
+                # ``NoResultsException``; the streaming path bypasses ``parse_response``,
+                # so mirror that rule explicitly here. Without it, an absent artifact
+                # would be written out as a successful *empty* file instead of
+                # surfacing "no results".
+                raise exceptions.NoResultsException(
+                    request,
+                    "The request returned no results.",
+                )
+
             pk = request.parser_kwargs or {}
             handle, name, created = request.result_parser.open_destination(
                 pk.get("folder"),

test/async_client_test.py

@@ -20,7 +20,6 @@
 import os
 import tempfile
 from contextlib import contextmanager
-
 import pytest
 import httpx
 import respx
@@ -410,12 +409,12 @@ async def test_async_known_good_lifecycle(self, uid):
             assert created.sources == ['nsrl']
             assert created.artifact_instance_id
             # A second feed flagging the same sha extends the same entry (no new row).
-            extended = await api.known_good_create(sha256=sha, source='winget')
+            extended = await api.known_good_create(sha256=sha, source='commercial')
             assert extended.id == created.id
-            assert extended.sources == ['nsrl', 'winget']
+            assert sorted(extended.sources) == ['commercial', 'nsrl']
             got = await api.known_good_get(sha256=sha)
             assert got.sha256 == sha
-            assert got.sources == ['nsrl', 'winget']
+            assert sorted(got.sources) == ['commercial', 'nsrl']
             deleted = await api.known_good_delete(sha256=sha)
             assert deleted.sha256 == sha
             with pytest.raises(exceptions.NotFoundException):
@@ -511,11 +510,25 @@ async def test_async_sample(self, uid):
             await _complete_sandbox_task(cape.id, 'cape')
             await _complete_sandbox_task(triage.id, 'triage')
 
+            # Poll until BOTH sandbox deps read COMPLETED *and* the LLM report was
+            # auto-triggered — the exact postconditions asserted below, not a proxy.
+            # Keying off llm_report alone raced: the report auto-triggers on *any
+            # one* completed dep (the scan or a single sandbox), so a response can
+            # show it triggered while sandbox_cape still projects NOT_TRIGGERED (the
+            # per-task projection doesn't update atomically). This test drives both
+            # sandboxes to SUCCEEDED, so both projections reach COMPLETED; waiting on
+            # them directly closes the window. See sync test_sample.
             _PRE_TRIGGER = {None, 'NOT_TRIGGERED', 'WAITING_FOR_OTHER_TASKS'}
             result = await api.sample(sha)
             for _ in range(90):
                 result = await api.sample(sha)
-                if result.tasks.get('llm_report', {}).get('requested_status') not in _PRE_TRIGGER:
+                tasks = result.tasks or {}
+                sandboxes_completed = all(
+                    tasks.get(f'sandbox_{s}', {}).get('requested_status') == 'COMPLETED'
+                    for s in ('cape', 'triage')
+                )
+                llm_triggered = tasks.get('llm_report', {}).get('requested_status') not in _PRE_TRIGGER
+                if sandboxes_completed and llm_triggered:
                     break
                 await asyncio.sleep(1)
         assert isinstance(result.artifact_instance, dict)
@@ -979,6 +992,28 @@ async def test_async_download():
         await api.aclose()
 
 
+@respx.mock
+async def test_async_download_204_raises_no_results():
+    """A 204 on a download means "no matching artifact" (the request worked but
+    returned nothing), so the streaming path must raise ``NoResultsException``
+    rather than write out a successful *empty* file — mirroring the shared
+    ``parse_response`` 204 rule that the streaming path otherwise bypasses."""
+    import tempfile, os
+
+    respx.get(f'{BASE_URL}/consumer/download/sha256/{SHA256}').mock(
+        return_value=httpx.Response(204))
+
+    api = PolySwarmAsyncAPI(API_KEY, uri=BASE_URL, community='gamma')
+    try:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            with pytest.raises(exceptions.NoResultsException):
+                await api.download(tmp_dir, SHA256)
+            # No empty artifact file should have been left behind.
+            assert os.listdir(tmp_dir) == []
+    finally:
+        await api.aclose()
+
+
 @respx.mock
 async def test_async_download_streams_in_chunks(monkeypatch):
     """Regression guard for the streaming-download fix: the body is consumed via
@@ -1015,17 +1050,18 @@ def write(self, b):
 
 
 @respx.mock
-async def test_async_exists_maps_2xx_true_404_false():
-    """End-to-end ``exists`` (the path the bot flagged as untested): the HEAD
-    status drives the result — any 2xx is True, 404 is False. Also locks the 2xx
-    generalisation (not a brittle ``== 200``)."""
+async def test_async_exists_maps_200_true_204_and_404_false():
+    """End-to-end ``exists``: the HEAD status drives the result. The endpoint
+    returns 200 when the artifact is present and 204 when it is absent ("request
+    worked, no matching artifact"), so only a 200 is True — a 204 is a successful
+    2xx that means the *opposite* of "exists" and must be False, as must a 404."""
     route = respx.head(f'{BASE_URL}/search/hash/sha256')
     api = PolySwarmAsyncAPI(API_KEY, uri=BASE_URL, community='gamma')
     try:
         route.mock(return_value=httpx.Response(200))
         assert await api.exists(SHA256) is True
-        route.mock(return_value=httpx.Response(204))   # 2xx-but-not-200 still "exists"
-        assert await api.exists(SHA256) is True
+        route.mock(return_value=httpx.Response(204))   # absent: "worked, nothing found"
+        assert await api.exists(SHA256) is False
         route.mock(return_value=httpx.Response(404))
         assert await api.exists(SHA256) is False
     finally:

test/client_scan_test.py

@@ -775,12 +775,12 @@ def test_known_good_lifecycle(self):
         assert created.sources == ['nsrl']
         assert created.artifact_instance_id
         # A second feed flagging the same sha extends the same entry (no new row).
-        extended = v3api.known_good_create(sha256=sha, source='winget')
+        extended = v3api.known_good_create(sha256=sha, source='commercial')
         assert extended.id == created.id
-        assert extended.sources == ['nsrl', 'winget']
+        assert sorted(extended.sources) == ['commercial', 'nsrl']
         got = v3api.known_good_get(sha256=sha)
         assert got.sha256 == sha
-        assert got.sources == ['nsrl', 'winget']
+        assert sorted(got.sources) == ['commercial', 'nsrl']
         deleted = v3api.known_good_delete(sha256=sha)
         assert deleted.sha256 == sha
         with pytest.raises(exceptions.NotFoundException):
@@ -880,18 +880,35 @@ def test_sample(self):
         _complete_sandbox_task(cape.id, 'cape')
         _complete_sandbox_task(triage.id, 'triage')
 
-        # Poll the sample until the LLM report has been auto-triggered. The view
-        # triggers it once a sandbox dep is COMPLETED, so the report's status
-        # moves NOT_TRIGGERED/WAITING_FOR_OTHER_TASKS -> PENDING (then FAILED here,
-        # since e2e has no OPENAI_API_KEY). We key off requested_status: the
-        # requested_id stays null until a report actually renders, which can't
-        # happen without an LLM. Reaching a triggered status also confirms the
-        # sandbox deps completed.
+        # Poll the sample until BOTH sandbox deps read COMPLETED *and* the LLM
+        # report has been auto-triggered — i.e. wait on the exact postconditions
+        # asserted below, not a proxy for them. The report's requested_status moves
+        # NOT_TRIGGERED/WAITING_FOR_OTHER_TASKS -> PENDING (then FAILED here, since
+        # e2e has no OPENAI_API_KEY); requested_id stays null until a report
+        # actually renders (impossible without an LLM), so requested_status is the
+        # trigger signal.
+        #
+        # Keying the loop off llm_report alone raced: the report auto-triggers as
+        # soon as *any one* dependency completes (the scan or a single sandbox), so
+        # a response can show it triggered while sandbox_cape still projects
+        # NOT_TRIGGERED — the delayed COLLECTING_DATA->SUCCEEDED transition (see
+        # _complete_sandbox_task) not yet folded into the per-task projection, which
+        # doesn't update atomically. That mismatch was the flake. This test drives
+        # BOTH sandboxes to SUCCEEDED before polling, so both projections do reach
+        # COMPLETED; waiting on them directly (not on the report as a proxy) closes
+        # the window. A 90x1s timeout here therefore means a dependency never
+        # settled — the scan's bounty window or a sandbox — not this loop.
         _PRE_TRIGGER = {None, 'NOT_TRIGGERED', 'WAITING_FOR_OTHER_TASKS'}
         result = api.sample(sha)
         for _ in range(90):
             result = api.sample(sha)
-            if result.tasks.get('llm_report', {}).get('requested_status') not in _PRE_TRIGGER:
+            tasks = result.tasks or {}
+            sandboxes_completed = all(
+                tasks.get(f'sandbox_{s}', {}).get('requested_status') == 'COMPLETED'
+                for s in ('cape', 'triage')
+            )
+            llm_triggered = tasks.get('llm_report', {}).get('requested_status') not in _PRE_TRIGGER
+            if sandboxes_completed and llm_triggered:
                 break
             time.sleep(1)
         assert isinstance(result.artifact_instance, dict)
@@ -903,3 +920,4 @@ def test_sample(self):
         # llm_report auto-triggered (PENDING -> ...); it cannot reach COMPLETED in
         # e2e (no LLM), so assert it was triggered, not finished.
         assert result.tasks['llm_report']['requested_status'] not in _PRE_TRIGGER
+