From c4a4bf2355f4d98a01c645fde2155432922ac872 Mon Sep 17 00:00:00 2001 From: Sumesh Premraj Date: Thu, 6 Jul 2023 12:13:22 +0200 Subject: [PATCH 1/3] fix headers passed into HttpAsyncHook --- airflow/providers/http/hooks/http.py | 2 +- tests/providers/http/hooks/test_http.py | 34 +++++++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/airflow/providers/http/hooks/http.py b/airflow/providers/http/hooks/http.py index 5bff4716a9542..0b0443efc484d 100644 --- a/airflow/providers/http/hooks/http.py +++ b/airflow/providers/http/hooks/http.py @@ -369,7 +369,7 @@ async def run( url, json=data if self.method in ("POST", "PATCH") else None, params=data if self.method == "GET" else None, - headers=headers, + headers=_headers, auth=auth, **extra_options, ) diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index 8d2bc8691d9a4..e702dded9c102 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -82,7 +82,6 @@ def test_get_request_with_port(self, mock_session, mock_request): ): expected_url = "http://test.com:1234/some/endpoint" for endpoint in ["some/endpoint", "/some/endpoint"]: - try: self.get_hook.run(endpoint) except MissingSchema: @@ -175,7 +174,6 @@ def test_post_request_do_not_raise_for_status_if_check_response_is_false(self, r @mock.patch("airflow.providers.http.hooks.http.requests.Session") def test_retry_on_conn_error(self, mocked_session): - retry_args = dict( wait=tenacity.wait_none(), stop=tenacity.stop_after_attempt(7), @@ -192,7 +190,6 @@ def send_and_raise(unused_request, **kwargs): assert self.get_hook._retry_obj.stop.max_attempt_number + 1 == mocked_session.call_count def test_run_with_advanced_retry(self, requests_mock): - requests_mock.get("http://test:8080/v1/test", status_code=200, reason="OK") retry_args = dict( @@ -298,7 +295,6 @@ def test_requests_ca_bundle_env_var(self, mock_session_send): with mock.patch( "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port ): - self.get_hook.run("/some/endpoint") mock_session_send.assert_called_once_with( @@ -317,7 +313,6 @@ def test_verify_respects_requests_ca_bundle_env_var(self, mock_session_send): with mock.patch( "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_port ): - self.get_hook.run("/some/endpoint", extra_options={"verify": True}) mock_session_send.assert_called_once_with( @@ -530,3 +525,32 @@ async def test_async_post_request_with_error_code(aioresponse): with mock.patch("airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection): with pytest.raises(AirflowException): await hook.run("v1/test") + + +@pytest.mark.asyncio +async def test_async_request_uses_connection_extra(aioresponse): + """Test api call asynchronously with a connection that has extra field.""" + + connection_extra = {"bareer": "test"} + connection_id = "http_default" + + def get_airflow_connection_with_extra(unused_conn_id=None): + return Connection( + conn_id=connection_id, conn_type="http", host="test:8080/", extra=json.dumps(connection_extra) + ) + + aioresponse.post( + "http://test:8080/v1/test", + status=200, + payload='{"status":{"status": 200}}', + reason="OK", + ) + + with mock.patch( + "airflow.hooks.base.BaseHook.get_connection", side_effect=get_airflow_connection_with_extra + ): + hook = HttpAsyncHook() + with mock.patch("aiohttp.ClientSession.post", new_callable=mock.AsyncMock) as mocked_function: + await hook.run("v1/test") + headers = mocked_function.call_args.kwargs.get("headers") + assert all(key in headers and headers[key] == value for key, value in connection_extra.items()) From d873c4c6e248c428eb779199de93db5422f5a445 Mon Sep 17 00:00:00 2001 From: Sumesh Premraj Date: Thu, 6 Jul 2023 19:56:14 +0200 Subject: [PATCH 2/3] Fix unit test for Python 3.7 --- setup.cfg | 1 + tests/providers/http/hooks/test_http.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 91d3b86331de0..4cc9f577657b1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -118,6 +118,7 @@ install_requires = markupsafe>=1.1.1 marshmallow-oneofschema>=2.0.1 mdit-py-plugins>=0.3.0 + mock>=5.0.2;python_version<="3.7" # Pip can not find a version that satisfies constraints if opentelemetry-api is not pinned. opentelemetry-api==1.15.0 opentelemetry-exporter-otlp diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index e702dded9c102..cf1169bde8cf6 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -21,9 +21,14 @@ import json import logging import os +import sys from collections import OrderedDict from http import HTTPStatus -from unittest import mock + +if sys.version_info.major == 3 and sys.version_info.minor >= 8: + from unittest import mock +else: + from unittest import mock import pytest import requests From 1ab58d12a9ee4661dbe5a47835c0e106afd57304 Mon Sep 17 00:00:00 2001 From: Sumesh Premraj Date: Thu, 6 Jul 2023 21:27:27 +0200 Subject: [PATCH 3/3] Revert "Fix unit test for Python 3.7" This reverts commit d873c4c6e248c428eb779199de93db5422f5a445. --- setup.cfg | 1 - tests/providers/http/hooks/test_http.py | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4cc9f577657b1..91d3b86331de0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -118,7 +118,6 @@ install_requires = markupsafe>=1.1.1 marshmallow-oneofschema>=2.0.1 mdit-py-plugins>=0.3.0 - mock>=5.0.2;python_version<="3.7" # Pip can not find a version that satisfies constraints if opentelemetry-api is not pinned. opentelemetry-api==1.15.0 opentelemetry-exporter-otlp diff --git a/tests/providers/http/hooks/test_http.py b/tests/providers/http/hooks/test_http.py index cf1169bde8cf6..e702dded9c102 100644 --- a/tests/providers/http/hooks/test_http.py +++ b/tests/providers/http/hooks/test_http.py @@ -21,14 +21,9 @@ import json import logging import os -import sys from collections import OrderedDict from http import HTTPStatus - -if sys.version_info.major == 3 and sys.version_info.minor >= 8: - from unittest import mock -else: - from unittest import mock +from unittest import mock import pytest import requests