From 64e45612b7995437045e756b1e36c3f835d1d66a Mon Sep 17 00:00:00 2001 From: Parker Hancock <633163+parkerhancock@users.noreply.github.com> Date: Thu, 7 Dec 2023 21:51:48 -0600 Subject: [PATCH] fixes for httpx --- tests/integration/test_httpx.py | 157 ++++++++++++++------------------ vcr/patch.py | 12 +-- vcr/stubs/httpx_stubs.py | 29 +----- 3 files changed, 77 insertions(+), 121 deletions(-) diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index 4425e02e0..45415a0e8 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -9,6 +9,14 @@ from vcr.stubs.httpx_stubs import HTTPX_REDIRECT_PARAM # noqa: E402 +@pytest.fixture(params=["https", "http"]) +def scheme(request): + """Fixture that returns both http and https.""" + return request.param + +@pytest.fixture +def httpbin(scheme): + return scheme + "://httpbin.org" class BaseDoRequest: _client_class = None @@ -16,6 +24,7 @@ class BaseDoRequest: def __init__(self, *args, **kwargs): self._client_args = args self._client_kwargs = kwargs + self._client_kwargs['follow_redirects'] = self._client_kwargs.get('follow_redirects', True) def _make_client(self): return self._client_class(*self._client_args, **self._client_kwargs) @@ -40,7 +49,10 @@ def client(self): def __call__(self, *args, **kwargs): return self.client.request(*args, timeout=60, **kwargs) - + + def stream(self, *args, **kwargs): + with self.client.stream(*args, **kwargs) as response: + return b"".join(response.iter_bytes()) class DoAsyncRequest(BaseDoRequest): _client_class = httpx.AsyncClient @@ -75,8 +87,22 @@ def __call__(self, *args, **kwargs): # Use one-time context and dispose of the loop/client afterwards with self: - return self(*args, **kwargs) + return self._loop.run_until_complete(self.client.request(*args, **kwargs)) + + async def _get_stream(self, *args, **kwargs): + async with self.client.stream(*args, **kwargs) as response: + content = b"" + async for c in response.aiter_bytes(): + content += c + return content + + def stream(self, *args, **kwargs): + if hasattr(self, "_loop"): + return self._loop.run_until_complete(self._get_stream(*args, **kwargs)) + # Use one-time context and dispose of the loop/client afterwards + with self: + return self._loop.run_until_complete(self._get_stream(*args, **kwargs)) def pytest_generate_tests(metafunc): if "do_request" in metafunc.fixturenames: @@ -89,8 +115,8 @@ def yml(tmpdir, request): @pytest.mark.online -def test_status(tmpdir, mockbin, do_request): - url = mockbin +def test_status(tmpdir, httpbin, do_request): + url = httpbin with vcr.use_cassette(str(tmpdir.join("status.yaml"))): response = do_request()("GET", url) @@ -102,8 +128,8 @@ def test_status(tmpdir, mockbin, do_request): @pytest.mark.online -def test_case_insensitive_headers(tmpdir, mockbin, do_request): - url = mockbin +def test_case_insensitive_headers(tmpdir, httpbin, do_request): + url = httpbin with vcr.use_cassette(str(tmpdir.join("whatever.yaml"))): do_request()("GET", url) @@ -116,8 +142,8 @@ def test_case_insensitive_headers(tmpdir, mockbin, do_request): @pytest.mark.online -def test_content(tmpdir, mockbin, do_request): - url = mockbin +def test_content(tmpdir, httpbin, do_request): + url = httpbin with vcr.use_cassette(str(tmpdir.join("cointent.yaml"))): response = do_request()("GET", url) @@ -129,23 +155,21 @@ def test_content(tmpdir, mockbin, do_request): @pytest.mark.online -def test_json(tmpdir, mockbin, do_request): - url = mockbin + "/request" - - headers = {"content-type": "application/json"} +def test_json(tmpdir, httpbin, do_request): + url = httpbin + "/json" with vcr.use_cassette(str(tmpdir.join("json.yaml"))): - response = do_request(headers=headers)("GET", url) + response = do_request()("GET", url) with vcr.use_cassette(str(tmpdir.join("json.yaml"))) as cassette: - cassette_response = do_request(headers=headers)("GET", url) + cassette_response = do_request()("GET", url) assert cassette_response.json() == response.json() assert cassette.play_count == 1 @pytest.mark.online -def test_params_same_url_distinct_params(tmpdir, mockbin, do_request): - url = mockbin + "/request" +def test_params_same_url_distinct_params(tmpdir, httpbin, do_request): + url = httpbin + "/get" headers = {"Content-Type": "application/json"} params = {"a": 1, "b": False, "c": "c"} @@ -165,22 +189,20 @@ def test_params_same_url_distinct_params(tmpdir, mockbin, do_request): @pytest.mark.online -def test_redirect(mockbin, yml, do_request): - url = mockbin + "/redirect/303/2" - - redirect_kwargs = {HTTPX_REDIRECT_PARAM.name: True} +def test_redirect(httpbin, yml, do_request): + url = httpbin + "/redirect-to" - response = do_request()("GET", url, **redirect_kwargs) + response = do_request()("GET", url) with vcr.use_cassette(yml): - response = do_request()("GET", url, **redirect_kwargs) + response = do_request()("GET", url, params={"url": "./get", "status_code": 302}) with vcr.use_cassette(yml) as cassette: - cassette_response = do_request()("GET", url, **redirect_kwargs) + cassette_response = do_request()("GET", url, params={"url": "./get", "status_code": 302}) assert cassette_response.status_code == response.status_code assert len(cassette_response.history) == len(response.history) - assert len(cassette) == 3 - assert cassette.play_count == 3 + assert len(cassette) == 2 + assert cassette.play_count == 2 # Assert that the real response and the cassette response have a similar # looking request_info. @@ -190,8 +212,8 @@ def test_redirect(mockbin, yml, do_request): @pytest.mark.online -def test_work_with_gzipped_data(mockbin, do_request, yml): - url = mockbin + "/gzip?foo=bar" +def test_work_with_gzipped_data(httpbin, do_request, yml): + url = httpbin + "/gzip?foo=bar" headers = {"accept-encoding": "deflate, gzip"} with vcr.use_cassette(yml): @@ -216,56 +238,32 @@ def test_simple_fetching(do_request, yml, url): assert str(cassette_response.request.url) == url assert cassette.play_count == 1 - -def test_behind_proxy(do_request): - # This is recorded because otherwise we should have a live proxy somewhere. - yml = ( - os.path.dirname(os.path.realpath(__file__)) + "/cassettes/" + "test_httpx_test_test_behind_proxy.yml" - ) - url = "https://mockbin.org/headers" - proxy = "http://localhost:8080" - proxies = {"http://": proxy, "https://": proxy} - - with vcr.use_cassette(yml): - response = do_request(proxies=proxies, verify=False)("GET", url) - - with vcr.use_cassette(yml) as cassette: - cassette_response = do_request(proxies=proxies, verify=False)("GET", url) - assert str(cassette_response.request.url) == url - assert cassette.play_count == 1 - - assert cassette_response.headers["Via"] == "my_own_proxy", str(cassette_response.headers) - assert cassette_response.request.url == response.request.url - - @pytest.mark.online -def test_cookies(tmpdir, mockbin, do_request): +def test_cookies(tmpdir, httpbin, do_request): def client_cookies(client): return list(client.client.cookies) def response_cookies(response): return list(response.cookies) - url = mockbin + "/bin/26148652-fe25-4f21-aaf5-689b5b4bf65f" - headers = {"cookie": "k1=v1;k2=v2"} + url = httpbin + "/cookies/set" + params = {"k1": "v1", "k2": "v2"} - with do_request(headers=headers) as client: + with do_request(params=params, follow_redirects=False) as client: assert client_cookies(client) == [] - redirect_kwargs = {HTTPX_REDIRECT_PARAM.name: True} - testfile = str(tmpdir.join("cookies.yml")) with vcr.use_cassette(testfile): - r1 = client("GET", url, **redirect_kwargs) + r1 = client("GET", url) assert response_cookies(r1) == ["k1", "k2"] - r2 = client("GET", url, **redirect_kwargs) + r2 = client("GET", url) assert response_cookies(r2) == ["k1", "k2"] assert client_cookies(client) == ["k1", "k2"] - with do_request(headers=headers) as new_client: + with do_request(params=params, follow_redirects=False) as new_client: assert client_cookies(new_client) == [] with vcr.use_cassette(testfile) as cassette: @@ -275,42 +273,19 @@ def response_cookies(response): assert response_cookies(cassette_response) == ["k1", "k2"] assert client_cookies(new_client) == ["k1", "k2"] - @pytest.mark.online -def test_relative_redirects(tmpdir, scheme, do_request, mockbin): - redirect_kwargs = {HTTPX_REDIRECT_PARAM.name: True} +def test_stream(tmpdir, httpbin, do_request): + url = httpbin + "/stream-bytes/512" + testfile = str(tmpdir.join("stream.yml")) - url = mockbin + "/redirect/301?to=/redirect/301?to=/request" - testfile = str(tmpdir.join("relative_redirects.yml")) with vcr.use_cassette(testfile): - response = do_request()("GET", url, **redirect_kwargs) - assert len(response.history) == 2, response - assert response.json()["url"].endswith("request") + response_content = do_request().stream("GET", url) + assert len(response_content) == 512 + with vcr.use_cassette(testfile) as cassette: - response = do_request()("GET", url, **redirect_kwargs) - assert len(response.history) == 2 - assert response.json()["url"].endswith("request") - - assert cassette.play_count == 3 - - -@pytest.mark.online -def test_redirect_wo_allow_redirects(do_request, mockbin, yml): - url = mockbin + "/redirect/308/5" - - redirect_kwargs = {HTTPX_REDIRECT_PARAM.name: False} - - with vcr.use_cassette(yml): - response = do_request()("GET", url, **redirect_kwargs) - - assert str(response.url).endswith("308/5") - assert response.status_code == 308 - - with vcr.use_cassette(yml) as cassette: - response = do_request()("GET", url, **redirect_kwargs) - - assert str(response.url).endswith("308/5") - assert response.status_code == 308 - + cassette_content = do_request().stream("GET", url) + assert cassette_content == response_content + assert len(cassette_content) == 512 assert cassette.play_count == 1 + \ No newline at end of file diff --git a/vcr/patch.py b/vcr/patch.py index afcaab571..f69ae7686 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -95,8 +95,8 @@ except ImportError: # pragma: no cover pass else: - _HttpxSyncClient_send = httpx.Client.send - _HttpxAsyncClient_send = httpx.AsyncClient.send + _HttpxSyncClient_send_single_request = httpx.Client._send_single_request + _HttpxAsyncClient_send_single_request = httpx.AsyncClient._send_single_request class CassettePatcherBuilder: @@ -307,11 +307,11 @@ def _httpx(self): else: from .stubs.httpx_stubs import async_vcr_send, sync_vcr_send - new_async_client_send = async_vcr_send(self._cassette, _HttpxAsyncClient_send) - yield httpx.AsyncClient, "send", new_async_client_send + new_async_client_send = async_vcr_send(self._cassette, _HttpxAsyncClient_send_single_request) + yield httpx.AsyncClient, "_send_single_request", new_async_client_send - new_sync_client_send = sync_vcr_send(self._cassette, _HttpxSyncClient_send) - yield httpx.Client, "send", new_sync_client_send + new_sync_client_send = sync_vcr_send(self._cassette, _HttpxSyncClient_send_single_request) + yield httpx.Client, "_send_single_request", new_sync_client_send def _urllib3_patchers(self, cpool, conn, stubs): http_connection_remover = ConnectionRemover( diff --git a/vcr/stubs/httpx_stubs.py b/vcr/stubs/httpx_stubs.py index 515855ee0..453d3290b 100644 --- a/vcr/stubs/httpx_stubs.py +++ b/vcr/stubs/httpx_stubs.py @@ -38,7 +38,7 @@ def _to_serialized_response(httpx_response): "status_code": httpx_response.status_code, "http_version": httpx_response.http_version, "headers": _transform_headers(httpx_response), - "content": httpx_response.content.decode("utf-8", "ignore"), + "content": httpx_response.content#.decode("utf-8", "ignore"), } @@ -57,7 +57,7 @@ def _from_serialized_headers(headers): @patch("httpx.Response.close", MagicMock()) @patch("httpx.Response.read", MagicMock()) def _from_serialized_response(request, serialized_response, history=None): - content = serialized_response.get("content").encode() + content = serialized_response.get("content") response = httpx.Response( status_code=serialized_response.get("status_code"), request=request, @@ -106,33 +106,12 @@ def _record_responses(cassette, vcr_request, real_response): def _play_responses(cassette, request, vcr_request, client, kwargs): - history = [] - - allow_redirects = kwargs.get( - HTTPX_REDIRECT_PARAM.name, - HTTPX_REDIRECT_PARAM.default, - ) vcr_response = cassette.play_response(vcr_request) response = _from_serialized_response(request, vcr_response) - - while allow_redirects and 300 <= response.status_code <= 399: - next_url = response.headers.get("location") - if not next_url: - break - - vcr_request = VcrRequest("GET", next_url, None, dict(response.headers)) - vcr_request = cassette.find_requests_with_most_matches(vcr_request)[0][0] - - history.append(response) - # add cookies from response to session cookie store - client.cookies.extract_cookies(response) - - vcr_response = cassette.play_response(vcr_request) - response = _from_serialized_response(vcr_request, vcr_response, history) - return response + async def _async_vcr_send(cassette, real_send, *args, **kwargs): vcr_request, response = _shared_vcr_send(cassette, real_send, *args, **kwargs) if response: @@ -141,6 +120,7 @@ async def _async_vcr_send(cassette, real_send, *args, **kwargs): return response real_response = await real_send(*args, **kwargs) + await real_response.aread() return _record_responses(cassette, vcr_request, real_response) @@ -160,6 +140,7 @@ def _sync_vcr_send(cassette, real_send, *args, **kwargs): return response real_response = real_send(*args, **kwargs) + real_response.read() return _record_responses(cassette, vcr_request, real_response)