From f871749bb168b179d6cd547305db8ac5da56511a Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Thu, 19 Dec 2024 20:27:30 -0500 Subject: [PATCH 1/7] Dune Table Create/Insert --- src/destinations/dune.py | 31 +++++++++++++----- src/interfaces.py | 3 ++ src/sources/dune.py | 20 ++---------- src/sources/postgres.py | 10 +++--- src/sources/type_maps.py | 46 +++++++++++++++++++++++++++ tests/unit/destinations_test.py | 56 ++++++++++++++++++++++++++------- 6 files changed, 125 insertions(+), 41 deletions(-) create mode 100644 src/sources/type_maps.py diff --git a/src/destinations/dune.py b/src/destinations/dune.py index 89d8603..aac084e 100644 --- a/src/destinations/dune.py +++ b/src/destinations/dune.py @@ -2,13 +2,12 @@ from dune_client.client import DuneClient from dune_client.models import DuneError -from pandas import DataFrame -from src.interfaces import Destination +from src.interfaces import Destination, TypedDataFrame from src.logger import log -class DuneDestination(Destination[DataFrame]): +class DuneDestination(Destination[TypedDataFrame]): """A class representing as Dune as a destination. Uses the Dune API to upload CSV data to a table. @@ -39,14 +38,14 @@ def validate(self) -> bool: """ return True - def save(self, data: DataFrame) -> int: - """Upload a DataFrame to Dune as a CSV. + def save(self, data: TypedDataFrame) -> int: + """Upload a TypedDataFrame to Dune as a CSV. Returns size of dataframe (i.e. number of "affected" rows). Parameters ---------- - data : DataFrame + data : TypedDataFrame The data to be uploaded to Dune, which will be converted to CSV format. Raises @@ -60,8 +59,26 @@ def save(self, data: DataFrame) -> int: """ try: + # TODO: Determine user name from DuneAPI key? + namespace = "username" + table_name = self.table_name log.debug("Uploading DF to Dune...") - result = self.client.upload_csv(self.table_name, data.to_csv(index=False)) + # TODO check first if table exists? Or warn if it did... + self.client.create_table( + namespace, + table_name, + schema=[ + {"name": name, "type": dtype, "nullable": "true"} + for name, dtype in data.types.items() + ], + ) + result = self.client.insert_table( + namespace, + table_name, + # TODO - bytes -> IO[bytes] + data=data.dataframe.to_csv(index=False), # type: ignore + content_type="text/csv", + ) if not result: raise RuntimeError("Dune Upload Failed") except DuneError as dune_e: diff --git a/src/interfaces.py b/src/interfaces.py index e499c8b..4967ec5 100644 --- a/src/interfaces.py +++ b/src/interfaces.py @@ -33,6 +33,9 @@ def is_empty(self) -> bool: return self.dataframe.empty +# TODO: maybe a good place to define schema transformations and other data manipulation? + + class Named(Protocol): """Represents any class with name field.""" diff --git a/src/sources/dune.py b/src/sources/dune.py index 2640738..e6145a1 100644 --- a/src/sources/dune.py +++ b/src/sources/dune.py @@ -13,35 +13,21 @@ from dune_client.query import QueryBase from dune_client.types import ParameterType, QueryParameter from pandas import DataFrame -from sqlalchemy import BIGINT, BOOLEAN, DATE, TIMESTAMP, VARCHAR +from sqlalchemy import VARCHAR from sqlalchemy.dialects.postgresql import ( - BYTEA, - DOUBLE_PRECISION, - INTEGER, JSONB, NUMERIC, ) from src.interfaces import Source, TypedDataFrame from src.logger import log +from src.sources.type_maps import DUNE_TO_PG DECIMAL_PATTERN = r"decimal\((\d+),\s*(\d+)\)" VARCHAR_PATTERN = r"varchar\((\d+)\)" -DUNE_TO_PG: dict[str, type[Any] | NUMERIC] = { - "bigint": BIGINT, - "integer": INTEGER, - "varbinary": BYTEA, - "date": DATE, - "boolean": BOOLEAN, - "varchar": VARCHAR, - "double": DOUBLE_PRECISION, - "real": DOUBLE_PRECISION, - "timestamp with time zone": TIMESTAMP, - "uint256": NUMERIC, -} - +# TODO - migrate type utilities to type_maps. def _parse_varchar_type(type_str: str) -> int | None: """Extract the length from Dune's varchar type string like varchar(255). diff --git a/src/sources/postgres.py b/src/sources/postgres.py index af92922..df804d7 100644 --- a/src/sources/postgres.py +++ b/src/sources/postgres.py @@ -121,21 +121,21 @@ async def fetch(self) -> TypedDataFrame: df = await loop.run_in_executor( None, lambda: pd.read_sql_query(self.query_string, con=self.engine) ) - # TODO include types. + # TODO - extract types and return TypedDataFrame. return TypedDataFrame(dataframe=_convert_bytea_to_hex(df), types={}) def is_empty(self, data: TypedDataFrame) -> bool: - """Check if the provided DataFrame is empty. + """Check if the provided TypedDataFrame is empty. Parameters ---------- - data : DataFrame - The DataFrame to check. + data : TypedDataFrame + The TypedDataFrame to check. Returns ------- bool - True if the DataFrame is empty, False otherwise. + True if the TypedDataFrame is empty, False otherwise. """ return data.is_empty() diff --git a/src/sources/type_maps.py b/src/sources/type_maps.py new file mode 100644 index 0000000..c953590 --- /dev/null +++ b/src/sources/type_maps.py @@ -0,0 +1,46 @@ +"""Data Type mappings.""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy import BIGINT, BOOLEAN, DATE, TIMESTAMP, VARCHAR +from sqlalchemy.dialects.postgresql import ( + BYTEA, + DOUBLE_PRECISION, + INTEGER, + NUMERIC, +) + +DECIMAL_PATTERN = r"decimal\((\d+),\s*(\d+)\)" +VARCHAR_PATTERN = r"varchar\((\d+)\)" + +DUNE_TO_PG: dict[str, type[Any] | NUMERIC] = { + "bigint": BIGINT, + "integer": INTEGER, + "varbinary": BYTEA, + "date": DATE, + "boolean": BOOLEAN, + "varchar": VARCHAR, + "double": DOUBLE_PRECISION, + "real": DOUBLE_PRECISION, + "timestamp with time zone": TIMESTAMP, + "uint256": NUMERIC, +} + +# https://docs.dune.com/api-reference/tables/endpoint/create#body-schema-type +# This map is not a perfect inverse of the one above. +# 1. Notice `DOUBLE_PRECISION` has two pre-images: we chose double +# 2. timestamp with time zone not aligned with timestamp +# 3. Apparently no JSONB support here. +PG_TO_DUNE: dict[type[Any] | NUMERIC, str] = { + BIGINT: "bigint", + INTEGER: "integer", + BYTEA: "varbinary", + DATE: "date", + BOOLEAN: "boolean", + VARCHAR: "varchar", + DOUBLE_PRECISION: "double", + TIMESTAMP: "timestamp", # This doesn't match with above + NUMERIC: "uint256", +} diff --git a/tests/unit/destinations_test.py b/tests/unit/destinations_test.py index 5d2a90d..22a0873 100644 --- a/tests/unit/destinations_test.py +++ b/tests/unit/destinations_test.py @@ -26,14 +26,32 @@ def setUpClass(cls): ) cls.env_patcher.start() - @patch("requests.sessions.Session.post") @patch("pandas.core.generic.NDFrame.to_csv", name="Fake csv writer") - def test_ensure_index_disabled_when_uploading(self, mock_to_csv, *_): - dummy_data = [ - {"foo": "bar"}, - {"baz": "daz"}, - ] - dummy_df = pd.DataFrame(dummy_data) + @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") + @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") + @patch("requests.sessions.Session.post") + def test_ensure_index_disabled_when_uploading( + self, mock_to_csv, mock_create_table, mock_insert_table, *_ + ): + mock_create_table.return_value = { + "namespace": "my_user", + "table_name": "my_data", + "full_name": "dune.my_user.my_data", + "example_query": "select * from dune.my_user.my_data", + "already_existed": False, + "message": "Table created successfully", + } + mock_insert_table.return_value = {"rows_written": 9000, "bytes_written": 90} + + dummy_df = TypedDataFrame( + dataframe=pd.DataFrame( + [ + {"foo": "bar"}, + {"baz": "daz"}, + ] + ), + types={"foo": "varchar", "baz": "varchar"}, + ) destination = DuneDestination( api_key=os.getenv("DUNE_API_KEY"), table_name="foo", @@ -53,10 +71,23 @@ def test_duneclient_sets_timeout(self, mock_to_csv, *_): assert destination.client.request_timeout == timeout @patch("dune_client.api.table.TableAPI.upload_csv", name="Fake CSV uploader") - def test_dune_error_handling(self, mock_dune_upload_csv): + @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") + @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") + def test_dune_error_handling( + self, mock_dune_upload_csv, mock_create_table, mock_insert_table + ): dest = DuneDestination(api_key="f00b4r", table_name="foo", request_timeout=10) df = pd.DataFrame([{"foo": "bar"}]) + mock_create_table.return_value = { + "namespace": "my_user", + "table_name": "my_data", + "full_name": "dune.my_user.my_data", + "example_query": "select * from dune.my_user.my_data", + "already_existed": False, + "message": "Table created successfully", + } + mock_insert_table.return_value = {"rows_written": 9000, "bytes_written": 90} dune_err = DuneError( data={"error": "bad stuff"}, response_class="response", @@ -67,8 +98,9 @@ def test_dune_error_handling(self, mock_dune_upload_csv): mock_dune_upload_csv.side_effect = dune_err + data = TypedDataFrame(df, {}) with self.assertLogs(level=ERROR) as logs: - dest.save(data=df) + dest.save(data) mock_dune_upload_csv.assert_called_once() @@ -83,7 +115,7 @@ def test_dune_error_handling(self, mock_dune_upload_csv): mock_dune_upload_csv.side_effect = val_err with self.assertLogs(level=ERROR) as logs: - dest.save(data=df) + dest.save(data) mock_dune_upload_csv.assert_called_once() expected_message = "Data processing error: Oops" @@ -92,7 +124,7 @@ def test_dune_error_handling(self, mock_dune_upload_csv): mock_dune_upload_csv.reset_mock() mock_dune_upload_csv.side_effect = runtime_err with self.assertLogs(level=ERROR) as logs: - dest.save(data=df) + dest.save(data) mock_dune_upload_csv.assert_called_once() expected_message = "Data processing error: Big Oops" @@ -106,7 +138,7 @@ def test_dune_error_handling(self, mock_dune_upload_csv): mock_dune_upload_csv.return_value = None with self.assertLogs(level=ERROR) as logs: - dest.save(data=df) + dest.save(data) mock_dune_upload_csv.assert_called_once() self.assertIn("Dune Upload Failed", logs.output[0]) From 5b956bfdbe07c27210094714e52f881bd976f06c Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Sat, 21 Dec 2024 20:20:10 -0500 Subject: [PATCH 2/7] Add Type Map and Fix Tests --- config.yaml | 2 +- src/destinations/dune.py | 20 ++++++++----- src/sources/postgres.py | 9 +++++- src/sources/type_maps.py | 25 +++++++++------- tests/unit/destinations_test.py | 53 +++++++++++++++++---------------- 5 files changed, 64 insertions(+), 45 deletions(-) diff --git a/config.yaml b/config.yaml index 8ceb06f..46f0279 100644 --- a/config.yaml +++ b/config.yaml @@ -36,7 +36,7 @@ jobs: query_string: "SELECT 1 as number, '\\x1234'::bytea as my_bytes;" destination: ref: Dune - table_name: dune_sync_test_table + table_name: bh2smith.dune_sync_test - name: cow-solvers source: diff --git a/src/destinations/dune.py b/src/destinations/dune.py index aac084e..a096bca 100644 --- a/src/destinations/dune.py +++ b/src/destinations/dune.py @@ -1,5 +1,7 @@ """Destination logic for Dune Analytics.""" +import io + from dune_client.client import DuneClient from dune_client.models import DuneError @@ -59,30 +61,34 @@ def save(self, data: TypedDataFrame) -> int: """ try: - # TODO: Determine user name from DuneAPI key? - namespace = "username" - table_name = self.table_name log.debug("Uploading DF to Dune...") + namespace, table_name = self._get_namespace_and_table_name() # TODO check first if table exists? Or warn if it did... self.client.create_table( namespace, table_name, schema=[ - {"name": name, "type": dtype, "nullable": "true"} - for name, dtype in data.types.items() + {"name": name, "type": dtype} for name, dtype in data.types.items() ], ) result = self.client.insert_table( namespace, table_name, - # TODO - bytes -> IO[bytes] - data=data.dataframe.to_csv(index=False), # type: ignore + data=io.BytesIO(data.dataframe.to_csv(index=False).encode()), content_type="text/csv", ) if not result: raise RuntimeError("Dune Upload Failed") + log.debug("Inserted DF to Dune, %s", result) except DuneError as dune_e: log.error("Dune did not accept our upload: %s", dune_e) except (ValueError, RuntimeError) as e: log.error("Data processing error: %s", e) return len(data) + + def _get_namespace_and_table_name(self) -> tuple[str, str]: + """Split the namespace, table name from the provided table name.""" + if "." not in self.table_name: + raise ValueError("Table name must be in the format namespace.table_name") + namespace, table_name = self.table_name.split(".") + return namespace, table_name diff --git a/src/sources/postgres.py b/src/sources/postgres.py index df804d7..41b40e0 100644 --- a/src/sources/postgres.py +++ b/src/sources/postgres.py @@ -11,6 +11,7 @@ from src.interfaces import Source, TypedDataFrame from src.logger import log +from src.sources.type_maps import PG_TO_DUNE def _convert_bytea_to_hex(df: DataFrame) -> DataFrame: @@ -118,11 +119,17 @@ async def fetch(self) -> TypedDataFrame: # of SQLAlchemy's synchronous interface. # The current solution using run_in_executor is a workaround # that moves the blocking operation to a thread pool. + # First get the column types + with self.engine.connect() as conn: + result = conn.execute(text(self.query_string)) + types = { + col.name: PG_TO_DUNE[col.type_code] for col in result.cursor.description + } df = await loop.run_in_executor( None, lambda: pd.read_sql_query(self.query_string, con=self.engine) ) # TODO - extract types and return TypedDataFrame. - return TypedDataFrame(dataframe=_convert_bytea_to_hex(df), types={}) + return TypedDataFrame(dataframe=_convert_bytea_to_hex(df), types=types) def is_empty(self, data: TypedDataFrame) -> bool: """Check if the provided TypedDataFrame is empty. diff --git a/src/sources/type_maps.py b/src/sources/type_maps.py index c953590..7c020e6 100644 --- a/src/sources/type_maps.py +++ b/src/sources/type_maps.py @@ -33,14 +33,19 @@ # 1. Notice `DOUBLE_PRECISION` has two pre-images: we chose double # 2. timestamp with time zone not aligned with timestamp # 3. Apparently no JSONB support here. -PG_TO_DUNE: dict[type[Any] | NUMERIC, str] = { - BIGINT: "bigint", - INTEGER: "integer", - BYTEA: "varbinary", - DATE: "date", - BOOLEAN: "boolean", - VARCHAR: "varchar", - DOUBLE_PRECISION: "double", - TIMESTAMP: "timestamp", # This doesn't match with above - NUMERIC: "uint256", +PG_TO_DUNE: dict[int, str] = { + 16: "boolean", + 17: "varbinary", + 20: "bigint", + 21: "bigint", # smallint + 23: "integer", + 25: "varchar", + # 26: "oid", + 700: "double", + 701: "double", + 1042: "varchar", + 1043: "varchar", + 1082: "timestamp", + 1114: "timestamp", # This doesn't match with above + 1700: "uint256", } diff --git a/tests/unit/destinations_test.py b/tests/unit/destinations_test.py index 22a0873..be3da9a 100644 --- a/tests/unit/destinations_test.py +++ b/tests/unit/destinations_test.py @@ -1,6 +1,6 @@ import os import unittest -from logging import ERROR, WARNING +from logging import DEBUG, ERROR, WARNING from unittest.mock import patch import pandas as pd @@ -26,12 +26,10 @@ def setUpClass(cls): ) cls.env_patcher.start() - @patch("pandas.core.generic.NDFrame.to_csv", name="Fake csv writer") @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") - @patch("requests.sessions.Session.post") def test_ensure_index_disabled_when_uploading( - self, mock_to_csv, mock_create_table, mock_insert_table, *_ + self, mock_create_table, mock_insert_table, *_ ): mock_create_table.return_value = { "namespace": "my_user", @@ -41,24 +39,28 @@ def test_ensure_index_disabled_when_uploading( "already_existed": False, "message": "Table created successfully", } + mock_insert_table.return_value = {"rows_written": 9000, "bytes_written": 90} dummy_df = TypedDataFrame( dataframe=pd.DataFrame( [ - {"foo": "bar"}, - {"baz": "daz"}, + {"foo": "bar", "baz": "one"}, + {"foo": "two", "baz": "two"}, ] ), types={"foo": "varchar", "baz": "varchar"}, ) destination = DuneDestination( api_key=os.getenv("DUNE_API_KEY"), - table_name="foo", + table_name="foo.bar", request_timeout=10, ) - destination.save(dummy_df) - mock_to_csv.assert_called_once_with(index=False) + with self.assertLogs(level=DEBUG) as logs: + destination.save(dummy_df) + + self.assertIn("Uploading DF to Dune", logs.output[0]) + self.assertIn("Inserted DF to Dune,", logs.output[1]) @patch("pandas.core.generic.NDFrame.to_csv", name="Fake csv writer") def test_duneclient_sets_timeout(self, mock_to_csv, *_): @@ -70,13 +72,12 @@ def test_duneclient_sets_timeout(self, mock_to_csv, *_): ) assert destination.client.request_timeout == timeout - @patch("dune_client.api.table.TableAPI.upload_csv", name="Fake CSV uploader") @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") - def test_dune_error_handling( - self, mock_dune_upload_csv, mock_create_table, mock_insert_table - ): - dest = DuneDestination(api_key="f00b4r", table_name="foo", request_timeout=10) + def test_dune_error_handling(self, mock_create_table, mock_insert_table): + dest = DuneDestination( + api_key="f00b4r", table_name="foo.bar", request_timeout=10 + ) df = pd.DataFrame([{"foo": "bar"}]) mock_create_table.return_value = { @@ -96,13 +97,13 @@ def test_dune_error_handling( val_err = ValueError("Oops") runtime_err = RuntimeError("Big Oops") - mock_dune_upload_csv.side_effect = dune_err + mock_create_table.side_effect = dune_err data = TypedDataFrame(df, {}) with self.assertLogs(level=ERROR) as logs: dest.save(data) - mock_dune_upload_csv.assert_called_once() + mock_create_table.assert_called_once() # does this shit really look better just because it's < 88 characters long? exmsg = ( @@ -111,36 +112,36 @@ def test_dune_error_handling( ) self.assertIn(exmsg, logs.output[0]) - mock_dune_upload_csv.reset_mock() - mock_dune_upload_csv.side_effect = val_err + mock_create_table.reset_mock() + mock_create_table.side_effect = val_err with self.assertLogs(level=ERROR) as logs: dest.save(data) - mock_dune_upload_csv.assert_called_once() + mock_create_table.assert_called_once() expected_message = "Data processing error: Oops" self.assertIn(expected_message, logs.output[0]) - mock_dune_upload_csv.reset_mock() - mock_dune_upload_csv.side_effect = runtime_err + mock_create_table.reset_mock() + mock_create_table.side_effect = runtime_err with self.assertLogs(level=ERROR) as logs: dest.save(data) - mock_dune_upload_csv.assert_called_once() + mock_create_table.assert_called_once() expected_message = "Data processing error: Big Oops" self.assertIn(expected_message, logs.output[0]) - mock_dune_upload_csv.reset_mock() + mock_create_table.reset_mock() # TIL: reset_mock() doesn't clear side effects.... - mock_dune_upload_csv.side_effect = None + mock_create_table.side_effect = None - mock_dune_upload_csv.return_value = None + mock_create_table.return_value = None with self.assertLogs(level=ERROR) as logs: dest.save(data) - mock_dune_upload_csv.assert_called_once() + mock_create_table.assert_called_once() self.assertIn("Dune Upload Failed", logs.output[0]) From b38d67e7fd461353ae85c5ed606ba570e044ba00 Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Tue, 7 Jan 2025 15:51:23 +0100 Subject: [PATCH 3/7] Fix bad merge --- tests/unit/destinations_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/unit/destinations_test.py b/tests/unit/destinations_test.py index ca1b06f..4552085 100644 --- a/tests/unit/destinations_test.py +++ b/tests/unit/destinations_test.py @@ -76,7 +76,9 @@ def test_duneclient_sets_timeout(self, mock_to_csv, *_): @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") def test_dune_error_handling(self, mock_create_table, mock_insert_table): - dest = DuneDestination(api_key="f00b4r", table_name="foo", request_timeout=10) + dest = DuneDestination( + api_key="f00b4r", table_name="foo.bar", request_timeout=10 + ) df = TypedDataFrame(pd.DataFrame([{"foo": "bar"}]), {}) mock_create_table.return_value = { @@ -98,9 +100,8 @@ def test_dune_error_handling(self, mock_create_table, mock_insert_table): mock_create_table.side_effect = dune_err - data = TypedDataFrame(df, {}) with self.assertLogs(level=ERROR) as logs: - dest.save(data) + dest.save(df) mock_create_table.assert_called_once() @@ -115,7 +116,7 @@ def test_dune_error_handling(self, mock_create_table, mock_insert_table): mock_create_table.side_effect = val_err with self.assertLogs(level=ERROR) as logs: - dest.save(data) + dest.save(df) mock_create_table.assert_called_once() expected_message = "Data processing error: Oops" @@ -124,7 +125,7 @@ def test_dune_error_handling(self, mock_create_table, mock_insert_table): mock_create_table.reset_mock() mock_create_table.side_effect = runtime_err with self.assertLogs(level=ERROR) as logs: - dest.save(data) + dest.save(df) mock_create_table.assert_called_once() expected_message = "Data processing error: Big Oops" @@ -138,7 +139,7 @@ def test_dune_error_handling(self, mock_create_table, mock_insert_table): mock_create_table.return_value = None with self.assertLogs(level=ERROR) as logs: - dest.save(data) + dest.save(df) mock_create_table.assert_called_once() self.assertIn("Dune Upload Failed", logs.output[0]) From f34c058c3620e2cd5c4e8b363e162e7fcff16d9e Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Sat, 11 Jan 2025 11:54:00 +0100 Subject: [PATCH 4/7] Skip Create & Updated Tests --- config.yaml | 1 + src/config.py | 5 ++- src/destinations/common.py | 5 +++ src/destinations/dune.py | 50 ++++++++++++++++++++++------- src/destinations/postgres.py | 3 +- tests/fixtures/config/basic.yaml | 2 +- tests/fixtures/config/sql_file.yaml | 2 +- tests/unit/config_test.py | 4 +-- tests/unit/destinations_test.py | 30 +++++++++++++++-- 9 files changed, 81 insertions(+), 21 deletions(-) create mode 100644 src/destinations/common.py diff --git a/config.yaml b/config.yaml index 7d73b1b..946e9f2 100644 --- a/config.yaml +++ b/config.yaml @@ -37,6 +37,7 @@ jobs: destination: ref: Dune table_name: bh2smith.dune_sync_test + if_exists: append - name: cow-solvers source: diff --git a/src/config.py b/src/config.py index 3d64c3d..0be9dad 100644 --- a/src/config.py +++ b/src/config.py @@ -305,6 +305,8 @@ def _build_destination( f' "{dest_config["ref"]}" defined in config' ) from e + if_exists = dest_config.get("if_exists", "append") + match dest.type: case Database.DUNE: try: @@ -323,13 +325,14 @@ def _build_destination( api_key=dest.key, table_name=dest_config["table_name"], request_timeout=request_timeout, + if_exists=if_exists, ) case Database.POSTGRES: return PostgresDestination( db_url=dest.key, table_name=dest_config["table_name"], - if_exists=dest_config.get("if_exists", "append"), + if_exists=if_exists, index_columns=dest_config.get("index_columns", []), ) raise ValueError(f"Unsupported destination_db type: {dest}") diff --git a/src/destinations/common.py b/src/destinations/common.py new file mode 100644 index 0000000..314882a --- /dev/null +++ b/src/destinations/common.py @@ -0,0 +1,5 @@ +"""Common structures used in multiple destination implementations.""" + +from typing import Literal + +TableExistsPolicy = Literal["append", "replace", "upsert", "insert_ignore"] diff --git a/src/destinations/dune.py b/src/destinations/dune.py index a096bca..ef44cde 100644 --- a/src/destinations/dune.py +++ b/src/destinations/dune.py @@ -3,8 +3,9 @@ import io from dune_client.client import DuneClient -from dune_client.models import DuneError +from dune_client.models import DuneError, QueryFailed +from src.destinations.common import TableExistsPolicy from src.interfaces import Destination, TypedDataFrame from src.logger import log @@ -28,9 +29,25 @@ class DuneDestination(Destination[TypedDataFrame]): """ - def __init__(self, api_key: str, table_name: str, request_timeout: int): + def __init__( + self, + api_key: str, + table_name: str, + request_timeout: int, + if_exists: TableExistsPolicy = "append", + ): self.client = DuneClient(api_key, request_timeout=request_timeout) + if "." not in table_name: + raise ValueError("Table name must be in the format namespace.table_name") + self.table_name: str = table_name + if if_exists not in {"append", "replace"}: + # TODO - Dune (support insert_ignore & upsert on table endpoints). + raise ValueError( + "Unsupported Table Existence Policy! " + "if_exists must be 'append' or 'replace'" + ) + self.if_exists: TableExistsPolicy = if_exists super().__init__() def validate(self) -> bool: @@ -63,14 +80,15 @@ def save(self, data: TypedDataFrame) -> int: try: log.debug("Uploading DF to Dune...") namespace, table_name = self._get_namespace_and_table_name() - # TODO check first if table exists? Or warn if it did... - self.client.create_table( - namespace, - table_name, - schema=[ - {"name": name, "type": dtype} for name, dtype in data.types.items() - ], - ) + if not self._skip_create(): + self.client.create_table( + namespace, + table_name, + schema=[ + {"name": name, "type": dtype} + for name, dtype in data.types.items() + ], + ) result = self.client.insert_table( namespace, table_name, @@ -86,9 +104,17 @@ def save(self, data: TypedDataFrame) -> int: log.error("Data processing error: %s", e) return len(data) + def _skip_create(self) -> bool: + return self.if_exists == "append" and self._table_exists() + + def _table_exists(self) -> bool: + try: + self.client.run_sql(f"SELECT count(*) FROM dune.{self.table_name}") + return True + except QueryFailed: + return False + def _get_namespace_and_table_name(self) -> tuple[str, str]: """Split the namespace, table name from the provided table name.""" - if "." not in self.table_name: - raise ValueError("Table name must be in the format namespace.table_name") namespace, table_name = self.table_name.split(".") return namespace, table_name diff --git a/src/destinations/postgres.py b/src/destinations/postgres.py index be1d2f1..0256809 100644 --- a/src/destinations/postgres.py +++ b/src/destinations/postgres.py @@ -11,11 +11,10 @@ ) from sqlalchemy.dialects.postgresql import insert +from src.destinations.common import TableExistsPolicy from src.interfaces import Destination, TypedDataFrame from src.logger import log -TableExistsPolicy = Literal["append", "replace", "upsert", "insert_ignore"] - class PostgresDestination(Destination[TypedDataFrame]): """A class representing PostgreSQL as a destination for data storage. diff --git a/tests/fixtures/config/basic.yaml b/tests/fixtures/config/basic.yaml index d72ca1c..5fa666b 100644 --- a/tests/fixtures/config/basic.yaml +++ b/tests/fixtures/config/basic.yaml @@ -37,4 +37,4 @@ jobs: query_string: SELECT 1; destination: ref: dune - table_name: table_name + table_name: table.name diff --git a/tests/fixtures/config/sql_file.yaml b/tests/fixtures/config/sql_file.yaml index a785e25..c05acf2 100644 --- a/tests/fixtures/config/sql_file.yaml +++ b/tests/fixtures/config/sql_file.yaml @@ -16,4 +16,4 @@ jobs: query_string: foo.sql destination: ref: dune - table_name: table_name + table_name: table.name diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index a3f8861..4abb8ef 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -55,7 +55,7 @@ def setUpClass(cls): "POLL_FREQUENCY_DUNE_PG": "192", "BLOCKCHAIN_NAME": "moosis", "WHAT_IF_EXISTS": "replace", - "table_name": "my_pg_table", + "table_name": "my.pg_table", }, clear=True, ) @@ -113,7 +113,7 @@ def test_load_templated_conf(self): self.assertEqual(int("192"), dune_to_pg_job.source.poll_frequency) self.assertEqual("moosis", dune_to_pg_job.source.query.params[0].value) self.assertEqual("replace", dune_to_pg_job.destination.if_exists) - self.assertEqual("my_pg_table", pg_to_dune_job.destination.table_name) + self.assertEqual("my.pg_table", pg_to_dune_job.destination.table_name) config_file = config_root / "basic_with_env_missing_vars.yaml" with self.assertRaises(KeyError): diff --git a/tests/unit/destinations_test.py b/tests/unit/destinations_test.py index 4552085..06f0ecb 100644 --- a/tests/unit/destinations_test.py +++ b/tests/unit/destinations_test.py @@ -26,6 +26,28 @@ def setUpClass(cls): ) cls.env_patcher.start() + def test_init_validation(self): + with self.assertRaises(ValueError) as ctx: + DuneDestination( + api_key="anything", + table_name="INVALID_TABLE_NAME", + request_timeout=10, + if_exists="replace", + ) + self.assertIn( + "Table name must be in the format namespace.table_name", + ctx.exception.args[0], + ) + + with self.assertRaises(ValueError) as ctx: + DuneDestination( + api_key="anything", + table_name="table.name", + request_timeout=10, + if_exists="upsert", + ) + self.assertIn("Unsupported Table Existence Policy!", ctx.exception.args[0]) + @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") def test_ensure_index_disabled_when_uploading( @@ -55,6 +77,7 @@ def test_ensure_index_disabled_when_uploading( api_key=os.getenv("DUNE_API_KEY"), table_name="foo.bar", request_timeout=10, + if_exists="replace", ) with self.assertLogs(level=DEBUG) as logs: @@ -68,7 +91,7 @@ def test_duneclient_sets_timeout(self, mock_to_csv, *_): for timeout in [1, 10, 100, 1000, 1500]: destination = DuneDestination( api_key=os.getenv("DUNE_API_KEY"), - table_name="foo", + table_name="foo.bar", request_timeout=timeout, ) assert destination.client.request_timeout == timeout @@ -77,7 +100,10 @@ def test_duneclient_sets_timeout(self, mock_to_csv, *_): @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") def test_dune_error_handling(self, mock_create_table, mock_insert_table): dest = DuneDestination( - api_key="f00b4r", table_name="foo.bar", request_timeout=10 + api_key="f00b4r", + table_name="foo.bar", + request_timeout=10, + if_exists="replace", ) df = TypedDataFrame(pd.DataFrame([{"foo": "bar"}]), {}) From 40d827e4f4f7d7f9d0e18593d839604811cec9df Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Sat, 11 Jan 2025 12:07:04 +0100 Subject: [PATCH 5/7] Test Table Exists --- tests/unit/destinations_test.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/unit/destinations_test.py b/tests/unit/destinations_test.py index 06f0ecb..bbe4191 100644 --- a/tests/unit/destinations_test.py +++ b/tests/unit/destinations_test.py @@ -1,11 +1,11 @@ import os import unittest from logging import DEBUG, ERROR, WARNING -from unittest.mock import patch +from unittest.mock import Mock, patch import pandas as pd import sqlalchemy -from dune_client.models import DuneError +from dune_client.models import DuneError, QueryFailed from src.destinations.dune import DuneDestination from src.destinations.postgres import PostgresDestination @@ -48,6 +48,21 @@ def test_init_validation(self): ) self.assertIn("Unsupported Table Existence Policy!", ctx.exception.args[0]) + def test_table_exists(self): + mock_client = Mock() + dest = DuneDestination( + api_key="anything", + table_name="table.name", + request_timeout=10, + if_exists="append", + ) + dest.client = mock_client + mock_client.run_sql.return_value = None # Not Raise! + self.assertEqual(True, dest._table_exists()) + + mock_client.run_sql.side_effect = QueryFailed("Table not found") + self.assertEqual(False, dest._table_exists()) + @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") def test_ensure_index_disabled_when_uploading( @@ -73,6 +88,7 @@ def test_ensure_index_disabled_when_uploading( ), types={"foo": "varchar", "baz": "varchar"}, ) + mock_client = Mock() destination = DuneDestination( api_key=os.getenv("DUNE_API_KEY"), table_name="foo.bar", From 73f4914ea440bd7c6873d094f74e143547902bb0 Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Sat, 11 Jan 2025 15:33:39 +0100 Subject: [PATCH 6/7] reintroduce upload_csv and unstable replace --- config.yaml | 9 +++- src/config.py | 6 +-- src/destinations/common.py | 4 -- src/destinations/dune.py | 82 ++++++++++++++++++++------------- src/destinations/postgres.py | 3 +- src/sources/postgres.py | 3 +- tests/unit/destinations_test.py | 52 ++++++++++++--------- 7 files changed, 94 insertions(+), 65 deletions(-) diff --git a/config.yaml b/config.yaml index 946e9f2..64f5047 100644 --- a/config.yaml +++ b/config.yaml @@ -33,11 +33,16 @@ jobs: - name: p2d-test source: ref: PG - query_string: "SELECT 1 as number, '\\x1234'::bytea as my_bytes;" + query_string: | + SELECT + 1 AS number, + '\\x1234'::bytea AS my_bytes, + '{"key": "value", "array": [1, 2, 3], "dict": {}}'::json AS my_json, + '[ [{"x": 1}, {"y": 2}], null, [{"z": 3}] ]'::json AS list_dict destination: ref: Dune table_name: bh2smith.dune_sync_test - if_exists: append + insertion_type: replace - name: cow-solvers source: diff --git a/src/config.py b/src/config.py index 0be9dad..788c250 100644 --- a/src/config.py +++ b/src/config.py @@ -305,8 +305,6 @@ def _build_destination( f' "{dest_config["ref"]}" defined in config' ) from e - if_exists = dest_config.get("if_exists", "append") - match dest.type: case Database.DUNE: try: @@ -325,14 +323,14 @@ def _build_destination( api_key=dest.key, table_name=dest_config["table_name"], request_timeout=request_timeout, - if_exists=if_exists, + insertion_type=dest_config.get("insertion_type", "append"), ) case Database.POSTGRES: return PostgresDestination( db_url=dest.key, table_name=dest_config["table_name"], - if_exists=if_exists, + if_exists=dest_config.get("if_exists", "append"), index_columns=dest_config.get("index_columns", []), ) raise ValueError(f"Unsupported destination_db type: {dest}") diff --git a/src/destinations/common.py b/src/destinations/common.py index 314882a..ed89f80 100644 --- a/src/destinations/common.py +++ b/src/destinations/common.py @@ -1,5 +1 @@ """Common structures used in multiple destination implementations.""" - -from typing import Literal - -TableExistsPolicy = Literal["append", "replace", "upsert", "insert_ignore"] diff --git a/src/destinations/dune.py b/src/destinations/dune.py index ef44cde..6be4ce9 100644 --- a/src/destinations/dune.py +++ b/src/destinations/dune.py @@ -1,14 +1,19 @@ """Destination logic for Dune Analytics.""" import io +from typing import Literal from dune_client.client import DuneClient from dune_client.models import DuneError, QueryFailed +from dune_client.query import QueryBase +from dune_client.types import QueryParameter +from pandas import DataFrame -from src.destinations.common import TableExistsPolicy from src.interfaces import Destination, TypedDataFrame from src.logger import log +InsertionPolicy = Literal["append", "replace", "upload_csv"] + class DuneDestination(Destination[TypedDataFrame]): """A class representing as Dune as a destination. @@ -34,20 +39,14 @@ def __init__( api_key: str, table_name: str, request_timeout: int, - if_exists: TableExistsPolicy = "append", + insertion_type: InsertionPolicy = "append", ): self.client = DuneClient(api_key, request_timeout=request_timeout) if "." not in table_name: raise ValueError("Table name must be in the format namespace.table_name") self.table_name: str = table_name - if if_exists not in {"append", "replace"}: - # TODO - Dune (support insert_ignore & upsert on table endpoints). - raise ValueError( - "Unsupported Table Existence Policy! " - "if_exists must be 'append' or 'replace'" - ) - self.if_exists: TableExistsPolicy = if_exists + self.insertion_type: InsertionPolicy = insertion_type super().__init__() def validate(self) -> bool: @@ -79,38 +78,57 @@ def save(self, data: TypedDataFrame) -> int: """ try: log.debug("Uploading DF to Dune...") - namespace, table_name = self._get_namespace_and_table_name() - if not self._skip_create(): - self.client.create_table( - namespace, - table_name, - schema=[ - {"name": name, "type": dtype} - for name, dtype in data.types.items() - ], - ) - result = self.client.insert_table( - namespace, - table_name, - data=io.BytesIO(data.dataframe.to_csv(index=False).encode()), - content_type="text/csv", - ) - if not result: - raise RuntimeError("Dune Upload Failed") - log.debug("Inserted DF to Dune, %s", result) + if self.insertion_type == "upload_csv": + self._upload_csv(data.dataframe) + else: + self._insert(data) + log.debug("Inserted DF to Dune, %s") except DuneError as dune_e: log.error("Dune did not accept our upload: %s", dune_e) except (ValueError, RuntimeError) as e: log.error("Data processing error: %s", e) return len(data) - def _skip_create(self) -> bool: - return self.if_exists == "append" and self._table_exists() + def _insert(self, data: TypedDataFrame) -> None: + namespace, table_name = self._get_namespace_and_table_name() + if self.insertion_type == "replace": + log.warning("Replacement feature is unstable!") + log.info("Deleting table: %s", table_name) + delete = self.client.delete_table(namespace, table_name) + log.info("Deleted: %s", delete) + + if not self._table_exists(): + log.info("Creating table: %s", self.table_name) + create = self.client.create_table( + namespace, + table_name, + schema=[ + {"name": name, "type": dtype} for name, dtype in data.types.items() + ], + ) + if not create: + raise RuntimeError("Dune Upload Failed") + log.info("Created: %s", create) + log.info("Inserting to: %s", self.table_name) + self.client.insert_table( + namespace, + table_name, + data=io.BytesIO(data.dataframe.to_csv(index=False).encode()), + content_type="text/csv", + ) + + def _upload_csv(self, data: DataFrame) -> None: + self.client.upload_csv(self.table_name, data.dataframe.to_csv(index=False)) def _table_exists(self) -> bool: try: - self.client.run_sql(f"SELECT count(*) FROM dune.{self.table_name}") - return True + results = self.client.run_query( + QueryBase( + 4554525, + params=[QueryParameter.text_type("table_name", self.table_name)], + ) + ) + return results.result is not None except QueryFailed: return False diff --git a/src/destinations/postgres.py b/src/destinations/postgres.py index 0256809..be1d2f1 100644 --- a/src/destinations/postgres.py +++ b/src/destinations/postgres.py @@ -11,10 +11,11 @@ ) from sqlalchemy.dialects.postgresql import insert -from src.destinations.common import TableExistsPolicy from src.interfaces import Destination, TypedDataFrame from src.logger import log +TableExistsPolicy = Literal["append", "replace", "upsert", "insert_ignore"] + class PostgresDestination(Destination[TypedDataFrame]): """A class representing PostgreSQL as a destination for data storage. diff --git a/src/sources/postgres.py b/src/sources/postgres.py index 05eee49..b4de256 100644 --- a/src/sources/postgres.py +++ b/src/sources/postgres.py @@ -142,7 +142,8 @@ async def fetch(self) -> TypedDataFrame: with self.engine.connect() as conn: result = conn.execute(text(self.query_string)) types = { - col.name: PG_TO_DUNE[col.type_code] for col in result.cursor.description + col.name: PG_TO_DUNE.get(col.type_code, "varchar") + for col in result.cursor.description } df = await loop.run_in_executor( None, lambda: pd.read_sql_query(self.query_string, con=self.engine) diff --git a/tests/unit/destinations_test.py b/tests/unit/destinations_test.py index bbe4191..5ff2767 100644 --- a/tests/unit/destinations_test.py +++ b/tests/unit/destinations_test.py @@ -32,41 +32,36 @@ def test_init_validation(self): api_key="anything", table_name="INVALID_TABLE_NAME", request_timeout=10, - if_exists="replace", + insertion_type="replace", ) self.assertIn( "Table name must be in the format namespace.table_name", ctx.exception.args[0], ) - with self.assertRaises(ValueError) as ctx: - DuneDestination( - api_key="anything", - table_name="table.name", - request_timeout=10, - if_exists="upsert", - ) - self.assertIn("Unsupported Table Existence Policy!", ctx.exception.args[0]) - def test_table_exists(self): mock_client = Mock() + dest = DuneDestination( api_key="anything", table_name="table.name", request_timeout=10, - if_exists="append", + insertion_type="append", ) dest.client = mock_client - mock_client.run_sql.return_value = None # Not Raise! + mock_result = Mock() + mock_result.result = "non-empty-result" + mock_client.run_query.return_value = mock_result self.assertEqual(True, dest._table_exists()) - mock_client.run_sql.side_effect = QueryFailed("Table not found") + mock_client.run_query.side_effect = QueryFailed("Table not found") self.assertEqual(False, dest._table_exists()) - @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") + @patch("dune_client.api.table.TableAPI.delete_table", name="Fake Table Deleter") @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") + @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") def test_ensure_index_disabled_when_uploading( - self, mock_create_table, mock_insert_table, *_ + self, mock_create_table, mock_insert_table, mock_delete_table, *_ ): mock_create_table.return_value = { "namespace": "my_user", @@ -76,7 +71,7 @@ def test_ensure_index_disabled_when_uploading( "already_existed": False, "message": "Table created successfully", } - + mock_delete_table.return_value = {"message": "Table deleted successfully"} mock_insert_table.return_value = {"rows_written": 9000, "bytes_written": 90} dummy_df = TypedDataFrame( @@ -88,19 +83,19 @@ def test_ensure_index_disabled_when_uploading( ), types={"foo": "varchar", "baz": "varchar"}, ) - mock_client = Mock() destination = DuneDestination( api_key=os.getenv("DUNE_API_KEY"), table_name="foo.bar", request_timeout=10, - if_exists="replace", + insertion_type="replace", ) + destination._table_exists = Mock(return_value=True) with self.assertLogs(level=DEBUG) as logs: destination.save(dummy_df) self.assertIn("Uploading DF to Dune", logs.output[0]) - self.assertIn("Inserted DF to Dune,", logs.output[1]) + self.assertIn("Inserted DF to Dune,", logs.output[-1]) @patch("pandas.core.generic.NDFrame.to_csv", name="Fake csv writer") def test_duneclient_sets_timeout(self, mock_to_csv, *_): @@ -114,13 +109,18 @@ def test_duneclient_sets_timeout(self, mock_to_csv, *_): @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") - def test_dune_error_handling(self, mock_create_table, mock_insert_table): + @patch("dune_client.api.table.TableAPI.delete_table", name="Fake Table Deleter") + def test_dune_error_handling( + self, mock_delete_table, mock_insert_table, mock_create_table, *_ + ): dest = DuneDestination( api_key="f00b4r", table_name="foo.bar", request_timeout=10, - if_exists="replace", + insertion_type="replace", ) + dest._table_exists = Mock(return_value=False) + df = TypedDataFrame(pd.DataFrame([{"foo": "bar"}]), {}) mock_create_table.return_value = { @@ -131,6 +131,9 @@ def test_dune_error_handling(self, mock_create_table, mock_insert_table): "already_existed": False, "message": "Table created successfully", } + mock_delete_table.return_value = { + "message": "Table bh2smith.dune_sync_test successfully deleted" + } mock_insert_table.return_value = {"rows_written": 9000, "bytes_written": 90} dune_err = DuneError( data={"error": "bad stuff"}, @@ -173,17 +176,24 @@ def test_dune_error_handling(self, mock_create_table, mock_insert_table): expected_message = "Data processing error: Big Oops" self.assertIn(expected_message, logs.output[0]) + # Reset all mocks to ensure clean state mock_create_table.reset_mock() + mock_insert_table.reset_mock() + mock_delete_table.reset_mock() # TIL: reset_mock() doesn't clear side effects.... mock_create_table.side_effect = None + mock_create_table.return_value = None + # Set return values explicitly mock_create_table.return_value = None with self.assertLogs(level=ERROR) as logs: dest.save(df) mock_create_table.assert_called_once() + mock_delete_table.assert_called_once() + self.assertIn("Dune Upload Failed", logs.output[0]) From 7af60c451898199a6c7aefd659defe497330b1dd Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Wed, 5 Feb 2025 14:05:45 +0100 Subject: [PATCH 7/7] Stash forgotten changes as commit --- poetry.lock | 84 ++++++++++++++++++++-- pyproject.toml | 30 ++++---- src/destinations/dune.py | 69 +++++++++--------- tests/unit/destinations_test.py | 119 +++++++++++++++----------------- 4 files changed, 182 insertions(+), 120 deletions(-) diff --git a/poetry.lock b/poetry.lock index 66540f0..20280c9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -6,6 +6,7 @@ version = "2.4.4" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "aiohappyeyeballs-2.4.4-py3-none-any.whl", hash = "sha256:a980909d50efcd44795c4afeca523296716d50cd756ddca6af8c65b996e27de8"}, {file = "aiohappyeyeballs-2.4.4.tar.gz", hash = "sha256:5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745"}, @@ -17,6 +18,7 @@ version = "3.11.10" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aiohttp-3.11.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cbad88a61fa743c5d283ad501b01c153820734118b65aee2bd7dbb735475ce0d"}, {file = "aiohttp-3.11.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80886dac673ceaef499de2f393fc80bb4481a129e6cb29e624a12e3296cc088f"}, @@ -114,6 +116,7 @@ version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, @@ -128,6 +131,7 @@ version = "3.3.5" description = "An abstract syntax tree for Python with inference support." optional = false python-versions = ">=3.9.0" +groups = ["dev"] files = [ {file = "astroid-3.3.5-py3-none-any.whl", hash = "sha256:a9d1c946ada25098d790e079ba2a1b112157278f3fb7e718ae6a9252f5835dc8"}, {file = "astroid-3.3.5.tar.gz", hash = "sha256:5cfc40ae9f68311075d27ef68a4841bdc5cc7f6cf86671b49f00607d30188e2d"}, @@ -139,6 +143,7 @@ version = "24.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, @@ -158,6 +163,7 @@ version = "24.10.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, @@ -202,6 +208,7 @@ version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, @@ -213,6 +220,7 @@ version = "3.4.0" description = "Validate configuration and produce human readable error messages." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, @@ -224,6 +232,7 @@ version = "3.4.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" +groups = ["main"] files = [ {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, @@ -338,6 +347,7 @@ version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, @@ -352,6 +362,8 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] +markers = "platform_system == \"Windows\" or sys_platform == \"win32\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -363,6 +375,7 @@ version = "7.6.9" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "coverage-7.6.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:85d9636f72e8991a1706b2b55b06c27545448baf9f6dbf51c4004609aacd7dcb"}, {file = "coverage-7.6.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:608a7fd78c67bee8936378299a6cb9f5149bb80238c7a566fc3e6717a4e68710"}, @@ -437,6 +450,7 @@ version = "0.6.7" description = "Easily serialize dataclasses to and from JSON." optional = false python-versions = "<4.0,>=3.7" +groups = ["main"] files = [ {file = "dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a"}, {file = "dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0"}, @@ -452,6 +466,7 @@ version = "1.2.15" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["main"] files = [ {file = "Deprecated-1.2.15-py2.py3-none-any.whl", hash = "sha256:353bc4a8ac4bfc96800ddab349d89c25dec1079f65fd53acdcc1e0b975b21320"}, {file = "deprecated-1.2.15.tar.gz", hash = "sha256:683e561a90de76239796e6b6feac66b99030d2dd3fcf61ef996330f14bbb9b0d"}, @@ -469,6 +484,7 @@ version = "0.3.9" description = "serialize all of Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "dill-0.3.9-py3-none-any.whl", hash = "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a"}, {file = "dill-0.3.9.tar.gz", hash = "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c"}, @@ -484,6 +500,7 @@ version = "0.3.9" description = "Distribution utilities" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"}, {file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"}, @@ -491,13 +508,14 @@ files = [ [[package]] name = "dune-client" -version = "1.7.7" +version = "1.7.8" description = "A simple framework for interacting with Dune Analytics official API service." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "dune_client-1.7.7-py3-none-any.whl", hash = "sha256:b698af8477ea1c6d82711641af91e7c4f09e18e9ba32dc1df3be3c30148f638a"}, - {file = "dune_client-1.7.7.tar.gz", hash = "sha256:b0b81fa3a62e4759692cab46b79b77f707bb608fef7218e559ad8732c715bf8b"}, + {file = "dune_client-1.7.8-py3-none-any.whl", hash = "sha256:0f6bf8a46afdd77face256a306c9377a18b67ca5f043beb4061ad5a706ad7a45"}, + {file = "dune_client-1.7.8.tar.gz", hash = "sha256:0afd76dfa7e6b5fd7b0058ebff585569e70cffb2ea5d97102ce1da1fcaac9d1b"}, ] [package.dependencies] @@ -519,6 +537,7 @@ version = "3.16.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, @@ -535,6 +554,7 @@ version = "1.5.0" description = "A list-like structure which implements collections.abc.MutableSequence" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a"}, {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb"}, @@ -636,6 +656,8 @@ version = "3.1.1" description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")" files = [ {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, @@ -722,6 +744,7 @@ version = "2.6.3" description = "File identification library for Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "identify-2.6.3-py2.py3-none-any.whl", hash = "sha256:9edba65473324c2ea9684b1f944fe3191db3345e50b6d04571d10ed164f8d7bd"}, {file = "identify-2.6.3.tar.gz", hash = "sha256:62f5dae9b5fef52c84cc188514e9ea4f3f636b1d8799ab5ebc475471f9e47a02"}, @@ -736,6 +759,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -750,6 +774,7 @@ version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -761,6 +786,7 @@ version = "5.13.2" description = "A Python utility / library to sort Python imports." optional = false python-versions = ">=3.8.0" +groups = ["dev"] files = [ {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, @@ -775,6 +801,7 @@ version = "3.23.1" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "marshmallow-3.23.1-py3-none-any.whl", hash = "sha256:fece2eb2c941180ea1b7fcbd4a83c51bfdd50093fdd3ad2585ee5e1df2508491"}, {file = "marshmallow-3.23.1.tar.gz", hash = "sha256:3a8dfda6edd8dcdbf216c0ede1d1e78d230a6dc9c5a088f58c4083b974a0d468"}, @@ -794,6 +821,7 @@ version = "0.7.0" description = "McCabe checker, plugin for flake8" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, @@ -805,6 +833,7 @@ version = "6.1.0" description = "multidict implementation" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"}, {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"}, @@ -906,6 +935,7 @@ version = "1.13.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, @@ -958,6 +988,7 @@ version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.5" +groups = ["main", "dev"] files = [ {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, @@ -969,6 +1000,7 @@ version = "0.3.1" description = "JsonDecoder for ndjson" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "ndjson-0.3.1-py2.py3-none-any.whl", hash = "sha256:839c22275e6baa3040077b83c005ac24199b94973309a8a1809be962c753a410"}, {file = "ndjson-0.3.1.tar.gz", hash = "sha256:bf9746cb6bb1cb53d172cda7f154c07c786d665ff28341e4e689b796b229e5d6"}, @@ -980,6 +1012,7 @@ version = "1.9.1" description = "Node.js virtual environment builder" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] files = [ {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, @@ -991,6 +1024,7 @@ version = "2.1.3" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.10" +groups = ["main", "dev"] files = [ {file = "numpy-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff"}, {file = "numpy-2.1.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5"}, @@ -1055,6 +1089,7 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -1066,6 +1101,7 @@ version = "2.2.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, @@ -1148,6 +1184,7 @@ version = "2.2.3.241126" description = "Type annotations for pandas" optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "pandas_stubs-2.2.3.241126-py3-none-any.whl", hash = "sha256:74aa79c167af374fe97068acc90776c0ebec5266a6e5c69fe11e9c2cf51f2267"}, {file = "pandas_stubs-2.2.3.241126.tar.gz", hash = "sha256:cf819383c6d9ae7d4dabf34cd47e1e45525bb2f312e6ad2939c2c204cb708acd"}, @@ -1163,6 +1200,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -1174,6 +1212,7 @@ version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, @@ -1190,6 +1229,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -1205,6 +1245,7 @@ version = "4.0.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pre_commit-4.0.1-py2.py3-none-any.whl", hash = "sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878"}, {file = "pre_commit-4.0.1.tar.gz", hash = "sha256:80905ac375958c0444c65e9cebebd948b3cdb518f335a091a670a89d652139d2"}, @@ -1223,6 +1264,7 @@ version = "0.21.1" description = "Python client for the Prometheus monitoring system." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "prometheus_client-0.21.1-py3-none-any.whl", hash = "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301"}, {file = "prometheus_client-0.21.1.tar.gz", hash = "sha256:252505a722ac04b0456be05c05f75f45d760c2911ffc45f2a06bcaed9f3ae3fb"}, @@ -1237,6 +1279,7 @@ version = "0.2.1" description = "Accelerated property cache" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "propcache-0.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6b3f39a85d671436ee3d12c017f8fdea38509e4f25b28eb25877293c98c243f6"}, {file = "propcache-0.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d51fbe4285d5db5d92a929e3e21536ea3dd43732c5b177c7ef03f918dff9f2"}, @@ -1328,6 +1371,7 @@ version = "2.9.10" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2"}, {file = "psycopg2_binary-2.9.10-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f"}, @@ -1404,6 +1448,7 @@ version = "3.3.2" description = "python code static checker" optional = false python-versions = ">=3.9.0" +groups = ["dev"] files = [ {file = "pylint-3.3.2-py3-none-any.whl", hash = "sha256:77f068c287d49b8683cd7c6e624243c74f92890f767f106ffa1ddf3c0a54cb7a"}, {file = "pylint-3.3.2.tar.gz", hash = "sha256:9ec054ec992cd05ad30a6df1676229739a73f8feeabf3912c995d17601052b01"}, @@ -1428,6 +1473,7 @@ version = "8.3.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, @@ -1448,6 +1494,7 @@ version = "0.24.0" description = "Pytest support for asyncio" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pytest_asyncio-0.24.0-py3-none-any.whl", hash = "sha256:a811296ed596b69bf0b6f3dc40f83bcaf341b155a269052d82efa2b25ac7037b"}, {file = "pytest_asyncio-0.24.0.tar.gz", hash = "sha256:d081d828e576d85f875399194281e92bf8a68d60d72d1a2faf2feddb6c46b276"}, @@ -1466,6 +1513,7 @@ version = "6.0.0" description = "Pytest plugin for measuring coverage." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pytest-cov-6.0.0.tar.gz", hash = "sha256:fde0b595ca248bb8e2d76f020b465f3b107c9632e6a1d1705f17834c89dcadc0"}, {file = "pytest_cov-6.0.0-py3-none-any.whl", hash = "sha256:eee6f1b9e61008bd34975a4d5bab25801eb31898b032dd55addc93e96fcaaa35"}, @@ -1484,6 +1532,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -1498,6 +1547,7 @@ version = "1.0.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, @@ -1512,6 +1562,7 @@ version = "2024.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, @@ -1523,6 +1574,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -1585,6 +1637,7 @@ version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, @@ -1606,6 +1659,7 @@ version = "0.8.2" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "ruff-0.8.2-py3-none-linux_armv6l.whl", hash = "sha256:c49ab4da37e7c457105aadfd2725e24305ff9bc908487a9bf8d548c6dad8bb3d"}, {file = "ruff-0.8.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ec016beb69ac16be416c435828be702ee694c0d722505f9c1f35e1b9c0cc1bf5"}, @@ -1633,6 +1687,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -1644,6 +1699,7 @@ version = "2.0.36" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59b8f3adb3971929a3e660337f5dacc5942c2cdb760afcabb2614ffbda9f9f72"}, {file = "SQLAlchemy-2.0.36-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37350015056a553e442ff672c2d20e6f4b6d0b2495691fa239d8aa18bb3bc908"}, @@ -1739,6 +1795,7 @@ version = "0.4" description = "SQLAlchemy stubs and mypy plugin" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "sqlalchemy-stubs-0.4.tar.gz", hash = "sha256:c665d6dd4482ef642f01027fa06c3d5e91befabb219dc71fc2a09e7d7695f7ae"}, {file = "sqlalchemy_stubs-0.4-py3-none-any.whl", hash = "sha256:5eec7aa110adf9b957b631799a72fef396b23ff99fe296df726645d01e312aa5"}, @@ -1754,6 +1811,7 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -1795,6 +1853,7 @@ version = "0.13.2" description = "Style preserving TOML library" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, @@ -1806,6 +1865,7 @@ version = "1.2.15.20241117" description = "Typing stubs for Deprecated" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "types-Deprecated-1.2.15.20241117.tar.gz", hash = "sha256:924002c8b7fddec51ba4949788a702411a2e3636cd9b2a33abd8ee119701d77e"}, {file = "types_Deprecated-1.2.15.20241117-py3-none-any.whl", hash = "sha256:a0cc5e39f769fc54089fd8e005416b55d74aa03f6964d2ed1a0b0b2e28751884"}, @@ -1817,6 +1877,7 @@ version = "2.9.0.20241206" description = "Typing stubs for python-dateutil" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "types_python_dateutil-2.9.0.20241206-py3-none-any.whl", hash = "sha256:e248a4bc70a486d3e3ec84d0dc30eec3a5f979d6e7ee4123ae043eedbb987f53"}, {file = "types_python_dateutil-2.9.0.20241206.tar.gz", hash = "sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb"}, @@ -1828,6 +1889,7 @@ version = "2024.2.0.20241003" description = "Typing stubs for pytz" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "types-pytz-2024.2.0.20241003.tar.gz", hash = "sha256:575dc38f385a922a212bac00a7d6d2e16e141132a3c955078f4a4fd13ed6cb44"}, {file = "types_pytz-2024.2.0.20241003-py3-none-any.whl", hash = "sha256:3e22df1336c0c6ad1d29163c8fda82736909eb977281cb823c57f8bae07118b7"}, @@ -1839,6 +1901,7 @@ version = "6.0.12.20240917" description = "Typing stubs for PyYAML" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"}, {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"}, @@ -1850,6 +1913,7 @@ version = "2.32.0.20241016" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"}, {file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"}, @@ -1864,6 +1928,7 @@ version = "75.6.0.20241126" description = "Typing stubs for setuptools" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "types_setuptools-75.6.0.20241126-py3-none-any.whl", hash = "sha256:aaae310a0e27033c1da8457d4d26ac673b0c8a0de7272d6d4708e263f2ea3b9b"}, {file = "types_setuptools-75.6.0.20241126.tar.gz", hash = "sha256:7bf25ad4be39740e469f9268b6beddda6e088891fa5a27e985c6ce68bf62ace0"}, @@ -1875,6 +1940,7 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -1886,6 +1952,7 @@ version = "0.9.0" description = "Runtime inspection utilities for typing module." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f"}, {file = "typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78"}, @@ -1901,6 +1968,7 @@ version = "2024.2" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" +groups = ["main"] files = [ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, @@ -1912,6 +1980,7 @@ version = "2.2.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, @@ -1929,6 +1998,7 @@ version = "20.28.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "virtualenv-20.28.0-py3-none-any.whl", hash = "sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0"}, {file = "virtualenv-20.28.0.tar.gz", hash = "sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa"}, @@ -1949,6 +2019,7 @@ version = "1.17.0" description = "Module for decorators, wrappers and monkey patching." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "wrapt-1.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a0c23b8319848426f305f9cb0c98a6e32ee68a36264f45948ccf8e7d2b941f8"}, {file = "wrapt-1.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1ca5f060e205f72bec57faae5bd817a1560fcfc4af03f414b08fa29106b7e2d"}, @@ -2023,6 +2094,7 @@ version = "1.18.3" description = "Yet another URL library" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34"}, {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7"}, @@ -2114,6 +2186,6 @@ multidict = ">=4.0" propcache = ">=0.2.0" [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = ">=3.12,<3.14" -content-hash = "8e35c0f43657930aa73e0bcb3df5fe4cb1a4dba53d1f58088f9133272b1b7510" +content-hash = "8ac8b88bec2d4be6cf9a166972830f059bb4e3f68ffbe33c0154cd4da6a07947" diff --git a/pyproject.toml b/pyproject.toml index 14f25d6..e6e4189 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,12 +9,12 @@ description = "" authors = [ "Benjamin Smith ", "mooster531 ", - "Felix Leupold " + "Felix Leupold ", ] [tool.poetry.dependencies] python = ">=3.12,<3.14" -dune-client = ">=1.7.7" +dune-client = ">=1.7.8" pandas = "*" sqlalchemy = "*" python-dotenv = "*" @@ -28,7 +28,7 @@ black = "*" pylint = "*" pre-commit = "*" pytest = "*" -pytest-cov="*" +pytest-cov = "*" pytest-asyncio = "*" pandas-stubs = "*" sqlalchemy-stubs = "*" @@ -58,17 +58,17 @@ exclude = [ [tool.ruff.lint] select = [ - "E", # pycodestyle errors - "F", # pyflakes - "I", # isort - "B", # flake8-bugbear - "C4", # flake8-comprehensions - "UP", # pyupgrade - "N", # pep8-naming - "D", # pydocstring + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade + "N", # pep8-naming + "D", # pydocstring "T20", # flake8-print "RET", # flake8-return - "PL", # pylint + "PL", # pylint ] ignore = [ @@ -80,11 +80,11 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "tests/**/*" = [ - "D", # Ignore all docstring rules in tests - "T20", # Allow print statements in tests + "D", # Ignore all docstring rules in tests + "T20", # Allow print statements in tests "N802", # Allow function names like "test_someFunction_does_something" "B018", # Allow using mock objects without explicit assert - "RET", # Allow multiple returns in test functions + "RET", # Allow multiple returns in test functions "E731", # Allow lambda assignments in tests ] diff --git a/src/destinations/dune.py b/src/destinations/dune.py index 6be4ce9..e502179 100644 --- a/src/destinations/dune.py +++ b/src/destinations/dune.py @@ -1,6 +1,7 @@ """Destination logic for Dune Analytics.""" import io +import sys from typing import Literal from dune_client.client import DuneClient @@ -76,49 +77,45 @@ def save(self, data: TypedDataFrame) -> int: For any data processing issues prior to the upload. """ - try: - log.debug("Uploading DF to Dune...") - if self.insertion_type == "upload_csv": - self._upload_csv(data.dataframe) - else: - self._insert(data) - log.debug("Inserted DF to Dune, %s") - except DuneError as dune_e: - log.error("Dune did not accept our upload: %s", dune_e) - except (ValueError, RuntimeError) as e: - log.error("Data processing error: %s", e) - return len(data) + log.debug("Uploading DF to Dune...") + if self.insertion_type == "upload_csv": + if not self._upload_csv(data.dataframe): + raise RuntimeError("Dune Upload Failed") + else: + self._insert(data) + log.debug("Inserted DF to Dune, %s") + + return len(data.dataframe) def _insert(self, data: TypedDataFrame) -> None: namespace, table_name = self._get_namespace_and_table_name() - if self.insertion_type == "replace": - log.warning("Replacement feature is unstable!") - log.info("Deleting table: %s", table_name) - delete = self.client.delete_table(namespace, table_name) - log.info("Deleted: %s", delete) - - if not self._table_exists(): - log.info("Creating table: %s", self.table_name) - create = self.client.create_table( + try: + if self.insertion_type == "replace": + log.debug("Clearing table: %s", table_name) + clear_result = self.client.clear_data(namespace, table_name) + log.debug("Cleared: %s", clear_result) + + log.info("Inserting to: %s", self.table_name) + self.client.insert_table( namespace, table_name, - schema=[ - {"name": name, "type": dtype} for name, dtype in data.types.items() - ], + data=io.BytesIO(data.dataframe.to_csv(index=False).encode()), + content_type="text/csv", ) - if not create: - raise RuntimeError("Dune Upload Failed") - log.info("Created: %s", create) - log.info("Inserting to: %s", self.table_name) - self.client.insert_table( - namespace, - table_name, - data=io.BytesIO(data.dataframe.to_csv(index=False).encode()), - content_type="text/csv", - ) + except DuneError as err: + if "This table was not found" in str(err): + api_ref = "https://docs.dune.com/api-reference/tables/endpoint/create" + log.error( + "Table %s doesn't exist. See %s for table creation details.", + self.table_name, + api_ref, + ) + raise err - def _upload_csv(self, data: DataFrame) -> None: - self.client.upload_csv(self.table_name, data.dataframe.to_csv(index=False)) + def _upload_csv(self, data: DataFrame) -> bool: + return self.client.upload_csv( + self.table_name, data.to_csv(index=False) + ) def _table_exists(self) -> bool: try: diff --git a/tests/unit/destinations_test.py b/tests/unit/destinations_test.py index 5ff2767..e0337ce 100644 --- a/tests/unit/destinations_test.py +++ b/tests/unit/destinations_test.py @@ -57,21 +57,14 @@ def test_table_exists(self): mock_client.run_query.side_effect = QueryFailed("Table not found") self.assertEqual(False, dest._table_exists()) - @patch("dune_client.api.table.TableAPI.delete_table", name="Fake Table Deleter") @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") - @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") + @patch("dune_client.api.table.TableAPI.clear_data", name="Fake Clearer") def test_ensure_index_disabled_when_uploading( - self, mock_create_table, mock_insert_table, mock_delete_table, *_ + self, mock_clear, mock_insert_table, *_ ): - mock_create_table.return_value = { - "namespace": "my_user", - "table_name": "my_data", - "full_name": "dune.my_user.my_data", - "example_query": "select * from dune.my_user.my_data", - "already_existed": False, - "message": "Table created successfully", + mock_clear.return_value = { + "message": "Table my_user.interest_rates successfully cleared" } - mock_delete_table.return_value = {"message": "Table deleted successfully"} mock_insert_table.return_value = {"rows_written": 9000, "bytes_written": 90} dummy_df = TypedDataFrame( @@ -107,12 +100,10 @@ def test_duneclient_sets_timeout(self, mock_to_csv, *_): ) assert destination.client.request_timeout == timeout - @patch("dune_client.api.table.TableAPI.create_table", name="Fake Table Creator") + @patch("dune_client.api.table.TableAPI.clear_data", name="Fake Data Clearer") + @patch("dune_client.api.table.TableAPI.upload_csv", name="Fake CSV Uploader") @patch("dune_client.api.table.TableAPI.insert_table", name="Fake Table Inserter") - @patch("dune_client.api.table.TableAPI.delete_table", name="Fake Table Deleter") - def test_dune_error_handling( - self, mock_delete_table, mock_insert_table, mock_create_table, *_ - ): + def test_dune_error_handling(self, mock_insert_table, mock_csv, mock_clear, *_): dest = DuneDestination( api_key="f00b4r", table_name="foo.bar", @@ -123,77 +114,79 @@ def test_dune_error_handling( df = TypedDataFrame(pd.DataFrame([{"foo": "bar"}]), {}) - mock_create_table.return_value = { - "namespace": "my_user", - "table_name": "my_data", - "full_name": "dune.my_user.my_data", - "example_query": "select * from dune.my_user.my_data", - "already_existed": False, - "message": "Table created successfully", - } - mock_delete_table.return_value = { - "message": "Table bh2smith.dune_sync_test successfully deleted" - } mock_insert_table.return_value = {"rows_written": 9000, "bytes_written": 90} - dune_err = DuneError( - data={"error": "bad stuff"}, + dune_not_exist_error = DuneError( + data={"error": "This table was not found"}, + response_class="response", + err=KeyError("you missed something"), + ) + dune_other_error = DuneError( + data={"error": "Bad Request"}, response_class="response", err=KeyError("you missed something"), ) val_err = ValueError("Oops") runtime_err = RuntimeError("Big Oops") - mock_create_table.side_effect = dune_err + mock_clear.side_effect = dune_not_exist_error - with self.assertLogs(level=ERROR) as logs: + with self.assertRaises(DuneError) as err: dest.save(df) - mock_create_table.assert_called_once() + self.assertEqual(err.exception, dune_not_exist_error) - # does this shit really look better just because it's < 88 characters long? - exmsg = ( - "Dune did not accept our upload: " - "Can't build response from {'error': 'bad stuff'}" - ) - self.assertIn(exmsg, logs.output[0]) + mock_clear.assert_called_once() - mock_create_table.reset_mock() - mock_create_table.side_effect = val_err - with self.assertLogs(level=ERROR) as logs: - dest.save(df) - - mock_create_table.assert_called_once() - expected_message = "Data processing error: Oops" - self.assertIn(expected_message, logs.output[0]) + mock_clear.reset_mock() + mock_clear.side_effect = dune_other_error - mock_create_table.reset_mock() - mock_create_table.side_effect = runtime_err - with self.assertLogs(level=ERROR) as logs: + with self.assertRaises(DuneError) as err: dest.save(df) - mock_create_table.assert_called_once() - expected_message = "Data processing error: Big Oops" - self.assertIn(expected_message, logs.output[0]) + mock_clear.assert_called_once() + + self.assertEqual(err.exception, dune_other_error) - # Reset all mocks to ensure clean state - mock_create_table.reset_mock() - mock_insert_table.reset_mock() - mock_delete_table.reset_mock() + mock_clear.reset_mock() + mock_clear.side_effect = val_err - # TIL: reset_mock() doesn't clear side effects.... - mock_create_table.side_effect = None - mock_create_table.return_value = None + with self.assertRaises(ValueError) as err: + dest.save(df) - # Set return values explicitly - mock_create_table.return_value = None + mock_clear.assert_called_once() + self.assertEqual(err.exception, val_err) + mock_clear.reset_mock() + mock_clear.side_effect = runtime_err with self.assertLogs(level=ERROR) as logs: dest.save(df) - mock_create_table.assert_called_once() - mock_delete_table.assert_called_once() + # Upload CSV: + dest.insertion_type = "upload_csv" + + mock_csv.return_value = False + + # mock_clear.assert_called_once() + # expected_message = "Data processing error: Big Oops" + # self.assertIn(expected_message, logs.output[0]) + # + # # Reset all mocks to ensure clean state + # mock_clear.reset_mock() + # mock_insert_table.reset_mock() + # + # # TIL: reset_mock() doesn't clear side effects.... + # mock_clear.side_effect = None + # mock_clear.return_value = None + # + # # Set return values explicitly + # mock_clear.return_value = None + # + with self.assertLogs(level=ERROR) as logs: + dest.save(df) + + mock_csv.assert_called_once() self.assertIn("Dune Upload Failed", logs.output[0])