From e0c414bafcd5ac2ed69a5f0a6d9968d0c8ce77ab Mon Sep 17 00:00:00 2001 From: eterna2 Date: Sun, 11 Oct 2020 20:02:52 +0800 Subject: [PATCH 1/3] WIP - add HttpStream --- e2fyi/utils/core/http.py | 161 ++++++++++++++++++++++++++++++++++ e2fyi/utils/core/http_test.py | 12 +++ poetry.lock | 25 ++++-- pyproject.toml | 2 + 4 files changed, 195 insertions(+), 5 deletions(-) create mode 100644 e2fyi/utils/core/http.py create mode 100644 e2fyi/utils/core/http_test.py diff --git a/e2fyi/utils/core/http.py b/e2fyi/utils/core/http.py new file mode 100644 index 0000000..d19505e --- /dev/null +++ b/e2fyi/utils/core/http.py @@ -0,0 +1,161 @@ +import io +import tempfile +import functools + +from typing import Tuple, Union, TypeVar, Iterator, Optional + +import requests + +from requests_toolbelt.streaming_iterator import StreamingIterator + +IntermediateObj = TypeVar("IntermediateObj") +StringOrBytes = TypeVar("StringOrBytes", str, bytes) + + +class HttpStream: + INMEM_SIZE: int = 0 + + def __init__( + self, + uri: str, + mode: str = "r", + buffering: int = -1, + encoding: str = None, + newline: str = None, + inmem_size: int = None, + decode_unicode: bool = False, + delimiter: Union[str, bytes] = None, + chunk_size: int = io.DEFAULT_BUFFER_SIZE, + **kwargs, + ): + self.uri = uri + self.mode = mode + self.buffering = buffering + self.newline = newline + self.encoding = encoding + + self._decode_unicode = decode_unicode + self._delimiter = delimiter + self._chunk_size = chunk_size + self._inmem_size = inmem_size + self._kwargs = kwargs + self._file = self._tempfile() + self._state: Optional[requests.Response] = None + + def _tempfile(self) -> tempfile.SpooledTemporaryFile: + return tempfile.SpooledTemporaryFile( + max_size=self._inmem_size or self.INMEM_SIZE, + mode="w+b" if "b" in self.mode else "w+", + buffering=self.buffering, + encoding=self.encoding, + newline=self.newline, + ) + + @property + def closed(self) -> bool: + return self._file.closed + + def seek(self, offset, whence: int = 0): + return self._file.seek(offset, whence) + + def tell(self) -> int: + return self._file.tell() + + def close(self): + return self._file.close() + + def write(self, data: Union[str, bytes, bytearray]) -> int: + return self._file.write(data) + + def is_empty(self) -> bool: + current = self.tell() + # go to the end of stream + self.seek(0, 2) + is_empty = self.tell() == 0 + self.seek(current) + return is_empty + + def read(self, size: Optional[int] = -1) -> Union[str, bytes, bytearray]: + if not self._state: + self._state = HttpStream._read2state(self.uri, **self._kwargs) + self._file = HttpStream._state2fileobj(self._state, self._tempfile()) + self.seek(0) + return self._file.read(size) # type: ignore + + def flush(self): + self._file.flush() + + @staticmethod + def _read2state(uri: str, **kwargs) -> requests.Response: + state = requests.get(uri, **kwargs) + state.raise_for_status() + return state + + @staticmethod + def _state2fileobj( + state: requests.Response, fileobj: tempfile.SpooledTemporaryFile + ) -> tempfile.SpooledTemporaryFile: + if "b" in fileobj.mode: + fileobj.write(state.content) + else: + fileobj.write(state.text) + return fileobj + + @classmethod + def _read2fileobj( + cls, uri: str, fileobj: tempfile.SpooledTemporaryFile, **kwargs + ) -> Tuple[requests.Response, tempfile.SpooledTemporaryFile]: + state = cls._read2state(uri, **kwargs) + file_ = cls._state2fileobj(state, fileobj) + file_.seek(0) + return state, file_ + + def _cleanup(self): + self._state = None + if self._file: + self._file.close() + + def __iter__(self) -> Iterator[Union[str, bytes, bytearray]]: + if not self._state: + self._state, self._file = HttpStream._read2fileobj( + self.uri, self._tempfile(), **self._kwargs + ) + if "b" in self.mode: + return self._state.iter_content( + chunk_size=self._chunk_size, decode_unicode=self._decode_unicode + ) + return self._state.iter_lines( + chunk_size=self._chunk_size, + decode_unicode=self._decode_unicode, + delimiter=self._delimiter, # type: ignore + ) + + def __enter__(self) -> "HttpStream": + if "w" in self.mode: + self._file.close() + self._file = self._tempfile() + return self + + if "a" in self.mode: + if not self._state: + self._file.close() + self._state, self._file = HttpStream._read2fileobj( + self.uri, self._tempfile(), **self._kwargs + ) + self.seek(0, 2) # go to end of stream + return self + + self.seek(0) + return self + + def __exit__(self, exc_type, exc_value, traceback): + try: + if "w" in self.mode or "a" in self.mode: + read = functools.partial(self._file.read, io.DEFAULT_BUFFER_SIZE) + iter_data = StreamingIterator(self.tell(), iter(read, "")) + self.seek(0) + resp = requests.post(self.uri, data=iter_data, **self._kwargs) + resp.raise_for_status() + finally: + self._file.close() + self._state = None diff --git a/e2fyi/utils/core/http_test.py b/e2fyi/utils/core/http_test.py new file mode 100644 index 0000000..b8c9556 --- /dev/null +++ b/e2fyi/utils/core/http_test.py @@ -0,0 +1,12 @@ +import unittest + +from unittest.mock import MagicMock, patch + +import e2fyi.utils.core.http + + +class HttpStreamTest(unittest.TestCase): + def test_http_stream(self): + + with e2fyi.utils.core.http.HttpStream("https://google.com") as stream: + self.assertEqual(stream.read(), "a") diff --git a/poetry.lock b/poetry.lock index 6e6e7a8..979fa99 100644 --- a/poetry.lock +++ b/poetry.lock @@ -124,7 +124,7 @@ python = "<3.4.0 || >=3.5.0" version = ">=1.20,<1.26" [[package]] -category = "dev" +category = "main" description = "Python package for providing Mozilla's CA Bundle." name = "certifi" optional = false @@ -132,7 +132,7 @@ python-versions = "*" version = "2020.6.20" [[package]] -category = "dev" +category = "main" description = "Universal encoding detector for Python 2 and 3" name = "chardet" optional = false @@ -280,7 +280,7 @@ version = "3.1.9" gitdb = ">=4.0.1,<5" [[package]] -category = "dev" +category = "main" description = "Internationalized Domain Names in Applications (IDNA)" name = "idna" optional = false @@ -595,7 +595,7 @@ python-versions = "*" version = "2020.9.27" [[package]] -category = "dev" +category = "main" description = "Python HTTP for Humans." name = "requests" optional = false @@ -612,6 +612,17 @@ urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] +[[package]] +category = "main" +description = "A utility belt for advanced users of python-requests" +name = "requests-toolbelt" +optional = false +python-versions = "*" +version = "0.9.1" + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" + [[package]] category = "main" description = "An Amazon S3 Transfer Manager" @@ -885,7 +896,7 @@ all = ["pandas"] pandas = ["pandas"] [metadata] -content-hash = "3903c265f7677b2b4b68d380679032c42ec4083de981bf729a7b132985b6ba8f" +content-hash = "a50ef825c0d1abd22261f0dba584644350937558862a39cca0a51bb77aecd42b" python-versions = ">=3.6.1,<4" [metadata.files] @@ -1284,6 +1295,10 @@ requests = [ {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, {file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"}, ] +requests-toolbelt = [ + {file = "requests-toolbelt-0.9.1.tar.gz", hash = "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"}, + {file = "requests_toolbelt-0.9.1-py2.py3-none-any.whl", hash = "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f"}, +] s3transfer = [ {file = "s3transfer-0.3.3-py2.py3-none-any.whl", hash = "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13"}, {file = "s3transfer-0.3.3.tar.gz", hash = "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db"}, diff --git a/pyproject.toml b/pyproject.toml index a7f56f7..e6e3889 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,8 @@ python-magic = {version = "0.4.*", markers = "sys_platform == 'linux'"} python-magic-bin = {version = "0.4.*", markers = "sys_platform == 'darwin' or sys_platform == 'windows'"} pydantic = ">=0.30" toml = "^0.10.1" +requests = "^2.24.0" +requests-toolbelt = "^0.9.1" [tool.poetry.extras] pandas = ["pandas"] From 4ee08c7aa25f89cd5463289eeacaa410e8c487c0 Mon Sep 17 00:00:00 2001 From: eterna2 Date: Sun, 11 Oct 2020 22:52:39 +0800 Subject: [PATCH 2/3] Add more unit test. Slowly converting to pytest instead. --- e2fyi/utils/core/http.py | 61 ++++++++-------- e2fyi/utils/core/http_test.py | 52 ++++++++++++-- poetry.lock | 128 +++++++++++++++++++++++++++++++++- pyproject.toml | 3 + 4 files changed, 210 insertions(+), 34 deletions(-) diff --git a/e2fyi/utils/core/http.py b/e2fyi/utils/core/http.py index d19505e..69cfefc 100644 --- a/e2fyi/utils/core/http.py +++ b/e2fyi/utils/core/http.py @@ -2,7 +2,7 @@ import tempfile import functools -from typing import Tuple, Union, TypeVar, Iterator, Optional +from typing import Union, TypeVar, Iterator, Optional import requests @@ -32,7 +32,7 @@ def __init__( self.mode = mode self.buffering = buffering self.newline = newline - self.encoding = encoding + self.encoding = encoding or "utf-8" self._decode_unicode = decode_unicode self._delimiter = delimiter @@ -76,10 +76,7 @@ def is_empty(self) -> bool: return is_empty def read(self, size: Optional[int] = -1) -> Union[str, bytes, bytearray]: - if not self._state: - self._state = HttpStream._read2state(self.uri, **self._kwargs) - self._file = HttpStream._state2fileobj(self._state, self._tempfile()) - self.seek(0) + self._read2fileobj() return self._file.read(size) # type: ignore def flush(self): @@ -101,30 +98,29 @@ def _state2fileobj( fileobj.write(state.text) return fileobj - @classmethod - def _read2fileobj( - cls, uri: str, fileobj: tempfile.SpooledTemporaryFile, **kwargs - ) -> Tuple[requests.Response, tempfile.SpooledTemporaryFile]: - state = cls._read2state(uri, **kwargs) - file_ = cls._state2fileobj(state, fileobj) - file_.seek(0) - return state, file_ + def _read2fileobj(self) -> requests.Response: + if not self._state: + if self._file: + self._file.close() + self._state = self._read2state(self.uri, **self._kwargs) + self._file = self._state2fileobj(self._state, self._tempfile()) + self._file.seek(0) + self.encoding = self._state.encoding or "utf-8" + return self._state def _cleanup(self): self._state = None if self._file: self._file.close() - def __iter__(self) -> Iterator[Union[str, bytes, bytearray]]: - if not self._state: - self._state, self._file = HttpStream._read2fileobj( - self.uri, self._tempfile(), **self._kwargs - ) + def __iter__(self) -> Iterator[bytes]: + kwargs: dict = {**self._kwargs, "stream": True} + response = requests.get(self.uri, **kwargs) if "b" in self.mode: - return self._state.iter_content( + return response.iter_content( chunk_size=self._chunk_size, decode_unicode=self._decode_unicode ) - return self._state.iter_lines( + return response.iter_lines( chunk_size=self._chunk_size, decode_unicode=self._decode_unicode, delimiter=self._delimiter, # type: ignore @@ -137,11 +133,7 @@ def __enter__(self) -> "HttpStream": return self if "a" in self.mode: - if not self._state: - self._file.close() - self._state, self._file = HttpStream._read2fileobj( - self.uri, self._tempfile(), **self._kwargs - ) + self._read2fileobj() self.seek(0, 2) # go to end of stream return self @@ -151,8 +143,21 @@ def __enter__(self) -> "HttpStream": def __exit__(self, exc_type, exc_value, traceback): try: if "w" in self.mode or "a" in self.mode: - read = functools.partial(self._file.read, io.DEFAULT_BUFFER_SIZE) - iter_data = StreamingIterator(self.tell(), iter(read, "")) + if "b" in self.mode: + iter_data = StreamingIterator( + self.tell(), self._file, encoding=self.encoding + ) + else: + + def iter_bytes(): + chunk = self._file.read(self._chunk_size) + if chunk: + yield chunk.encode(self.encoding) + + iter_data = StreamingIterator( + self.tell(), iter(iter_bytes()), encoding=self.encoding + ) + self.seek(0) resp = requests.post(self.uri, data=iter_data, **self._kwargs) resp.raise_for_status() diff --git a/e2fyi/utils/core/http_test.py b/e2fyi/utils/core/http_test.py index b8c9556..e219042 100644 --- a/e2fyi/utils/core/http_test.py +++ b/e2fyi/utils/core/http_test.py @@ -2,11 +2,55 @@ from unittest.mock import MagicMock, patch -import e2fyi.utils.core.http +import requests_mock + +from e2fyi.utils.core.http import HttpStream class HttpStreamTest(unittest.TestCase): - def test_http_stream(self): + def test_http_stream_read_text(self): + + with requests_mock.Mocker() as mock: + mocked_url = "https://foo.bar" + expected_text = "hello\nworld\n" + mock.get(mocked_url, text=expected_text) + + self.assertEqual(HttpStream(mocked_url).read(), expected_text) + self.assertEqual(list(HttpStream(mocked_url)), [b"hello", b"world"]) + + with HttpStream("https://foo.bar") as stream: + self.assertEqual(stream.read(), expected_text) + + def test_http_stream_read_bin(self): + + with requests_mock.Mocker() as mock: + mocked_url = "https://foo.bar" + expected_content = b"hello\nworld\n" + mock.get(mocked_url, content=expected_content) + + self.assertEqual(HttpStream(mocked_url, mode="rb").read(), expected_content) + self.assertEqual( + list(HttpStream(mocked_url, mode="rb", chunk_size=5)), + [b"hello", b"\nworl", b"d\n"], + ) + + with HttpStream("https://foo.bar", mode="rb") as stream: + self.assertEqual(stream.read(), expected_content) + + def test_http_stream_write_text(self): + + streamed_content = None + + def side_effect(*args, **kwargs): + nonlocal streamed_content + if "data" in kwargs: + streamed_content = kwargs["data"].read() + return MagicMock() + + with patch("requests.post", side_effect=side_effect) as mock_post: + + with HttpStream("https://foo.bar", mode="w") as stream: + stream.write("line1") + stream.write("line2") - with e2fyi.utils.core.http.HttpStream("https://google.com") as stream: - self.assertEqual(stream.read(), "a") + assert streamed_content == b"line1line2" diff --git a/poetry.lock b/poetry.lock index 979fa99..9ed0824 100644 --- a/poetry.lock +++ b/poetry.lock @@ -32,10 +32,18 @@ wrapt = ">=1.11,<2.0" python = "<3.8" version = ">=1.4.0,<1.5" +[[package]] +category = "dev" +description = "Atomic file writes." +marker = "sys_platform == \"win32\"" +name = "atomicwrites" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.4.0" + [[package]] category = "dev" description = "Classes Without Boilerplate" -marker = "platform_python_implementation == \"CPython\"" name = "attrs" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" @@ -311,6 +319,14 @@ zipp = ">=0.5" docs = ["sphinx", "rst.linker"] testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] +[[package]] +category = "dev" +description = "iniconfig: brain-dead simple config-ini parsing" +name = "iniconfig" +optional = false +python-versions = "*" +version = "1.0.1" + [[package]] category = "dev" description = "A Python utility / library to sort Python imports." @@ -475,6 +491,30 @@ optional = false python-versions = ">=2.6" version = "5.5.0" +[[package]] +category = "dev" +description = "plugin and hook calling mechanisms for python" +name = "pluggy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.13.1" + +[package.dependencies] +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=0.12" + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +category = "dev" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +name = "py" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.9.0" + [[package]] category = "dev" description = "Python style guide checker" @@ -540,6 +580,46 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" version = "2.4.7" +[[package]] +category = "dev" +description = "pytest: simple powerful testing with Python" +name = "pytest" +optional = false +python-versions = ">=3.5" +version = "6.1.1" + +[package.dependencies] +atomicwrites = ">=1.0" +attrs = ">=17.4.0" +colorama = "*" +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.8.2" +toml = "*" + +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=0.12" + +[package.extras] +checkqa_mypy = ["mypy (0.780)"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +category = "dev" +description = "Thin-wrapper around the mock package for easier use with pytest" +name = "pytest-mock" +optional = false +python-versions = ">=3.5" +version = "3.3.1" + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "tox", "pytest-asyncio"] + [[package]] category = "main" description = "Extensions to the standard Python datetime module" @@ -612,6 +692,22 @@ urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] +[[package]] +category = "dev" +description = "Mock out responses from the requests package" +name = "requests-mock" +optional = false +python-versions = "*" +version = "1.8.0" + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] + [[package]] category = "main" description = "A utility belt for advanced users of python-requests" @@ -896,7 +992,7 @@ all = ["pandas"] pandas = ["pandas"] [metadata] -content-hash = "a50ef825c0d1abd22261f0dba584644350937558862a39cca0a51bb77aecd42b" +content-hash = "854f055d13bb9d4238ed2661f7da3217fed306b30a2143157b31fb64baa49570" python-versions = ">=3.6.1,<4" [metadata.files] @@ -912,6 +1008,10 @@ astroid = [ {file = "astroid-2.4.2-py3-none-any.whl", hash = "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386"}, {file = "astroid-2.4.2.tar.gz", hash = "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703"}, ] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] attrs = [ {file = "attrs-20.2.0-py2.py3-none-any.whl", hash = "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"}, {file = "attrs-20.2.0.tar.gz", hash = "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594"}, @@ -1035,6 +1135,10 @@ importlib-metadata = [ {file = "importlib_metadata-1.7.0-py2.py3-none-any.whl", hash = "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070"}, {file = "importlib_metadata-1.7.0.tar.gz", hash = "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83"}, ] +iniconfig = [ + {file = "iniconfig-1.0.1-py3-none-any.whl", hash = "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437"}, + {file = "iniconfig-1.0.1.tar.gz", hash = "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"}, +] isort = [ {file = "isort-5.5.4-py3-none-any.whl", hash = "sha256:36f0c6659b9000597e92618d05b72d4181104cf59472b1c6a039e3783f930c95"}, {file = "isort-5.5.4.tar.gz", hash = "sha256:ba040c24d20aa302f78f4747df549573ae1eaf8e1084269199154da9c483f07f"}, @@ -1199,6 +1303,14 @@ pbr = [ {file = "pbr-5.5.0-py2.py3-none-any.whl", hash = "sha256:5adc0f9fc64319d8df5ca1e4e06eea674c26b80e6f00c530b18ce6a6592ead15"}, {file = "pbr-5.5.0.tar.gz", hash = "sha256:14bfd98f51c78a3dd22a1ef45cf194ad79eee4a19e8e1a0d5c7f8e81ffe182ea"}, ] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +py = [ + {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, + {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, +] pycodestyle = [ {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, @@ -1238,6 +1350,14 @@ pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] +pytest = [ + {file = "pytest-6.1.1-py3-none-any.whl", hash = "sha256:7a8190790c17d79a11f847fba0b004ee9a8122582ebff4729a082c109e81a4c9"}, + {file = "pytest-6.1.1.tar.gz", hash = "sha256:8f593023c1a0f916110285b6efd7f99db07d59546e3d8c36fc60e2ab05d3be92"}, +] +pytest-mock = [ + {file = "pytest-mock-3.3.1.tar.gz", hash = "sha256:a4d6d37329e4a893e77d9ffa89e838dd2b45d5dc099984cf03c703ac8411bb82"}, + {file = "pytest_mock-3.3.1-py3-none-any.whl", hash = "sha256:024e405ad382646318c4281948aadf6fe1135632bea9cc67366ea0c4098ef5f2"}, +] python-dateutil = [ {file = "python-dateutil-2.8.0.tar.gz", hash = "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"}, {file = "python_dateutil-2.8.0-py2.py3-none-any.whl", hash = "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb"}, @@ -1295,6 +1415,10 @@ requests = [ {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, {file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"}, ] +requests-mock = [ + {file = "requests-mock-1.8.0.tar.gz", hash = "sha256:e68f46844e4cee9d447150343c9ae875f99fa8037c6dcf5f15bf1fe9ab43d226"}, + {file = "requests_mock-1.8.0-py2.py3-none-any.whl", hash = "sha256:11215c6f4df72702aa357f205cf1e537cffd7392b3e787b58239bde5fb3db53b"}, +] requests-toolbelt = [ {file = "requests-toolbelt-0.9.1.tar.gz", hash = "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"}, {file = "requests_toolbelt-0.9.1-py2.py3-none-any.whl", hash = "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f"}, diff --git a/pyproject.toml b/pyproject.toml index e6e3889..8b59e76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ sphinx-autoapi = "^1.5.0" coverage = "^5.3" coveralls = "^2.1.2" m2r2 = "^0.2.5" +requests_mock = "^1.8.0" +pytest = "^6.1.1" +pytest-mock = "^3.3.1" [build-system] requires = ["poetry>=0.12"] From 6146443a46c9dbe13643a16e01e58962c85df81b Mon Sep 17 00:00:00 2001 From: eterna2 Date: Sun, 11 Oct 2020 23:11:53 +0800 Subject: [PATCH 3/3] fix some of the lint issues --- .pylintrc | 2 +- e2fyi/utils/core/http.py | 5 ++--- e2fyi/utils/core/http_test.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.pylintrc b/.pylintrc index 03638f6..b5d9dc8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -13,7 +13,7 @@ ignore=CVS # Add files or directories matching the regex patterns to the blacklist. The # regex matches against base names, not paths. -ignore-patterns=setup.py,test.py,rst/*.py +ignore-patterns=test.py,.*_test.py # Pickle collected data for later comparisons. persistent=yes diff --git a/e2fyi/utils/core/http.py b/e2fyi/utils/core/http.py index 69cfefc..adfd00f 100644 --- a/e2fyi/utils/core/http.py +++ b/e2fyi/utils/core/http.py @@ -1,6 +1,5 @@ import io import tempfile -import functools from typing import Union, TypeVar, Iterator, Optional @@ -12,10 +11,10 @@ StringOrBytes = TypeVar("StringOrBytes", str, bytes) -class HttpStream: +class HttpStream(io.IOBase): # pylint: disable=too-many-instance-attributes INMEM_SIZE: int = 0 - def __init__( + def __init__( # pylint: disable=super-init-not-called self, uri: str, mode: str = "r", diff --git a/e2fyi/utils/core/http_test.py b/e2fyi/utils/core/http_test.py index e219042..78b1d30 100644 --- a/e2fyi/utils/core/http_test.py +++ b/e2fyi/utils/core/http_test.py @@ -41,13 +41,13 @@ def test_http_stream_write_text(self): streamed_content = None - def side_effect(*args, **kwargs): + def side_effect(*_, **kwargs): nonlocal streamed_content if "data" in kwargs: streamed_content = kwargs["data"].read() return MagicMock() - with patch("requests.post", side_effect=side_effect) as mock_post: + with patch("requests.post", side_effect=side_effect): with HttpStream("https://foo.bar", mode="w") as stream: stream.write("line1")