From e8e7c9dccbfd05ca5a7f3388e3e01ecd26cc9443 Mon Sep 17 00:00:00 2001 From: I335851 Date: Wed, 9 Mar 2022 14:26:51 +0100 Subject: [PATCH 01/17] tests: add pytest-aiohttp plugin --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 1614c06c..77e85c8b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,6 @@ requests==2.23.0 pytest>=4.6.0 +pytest-aiohttp>=1.0.4 responses>=0.8.1 setuptools>=38.2.4 setuptools-scm>=1.15.6 From 08a7a7f192e3326471734c7b820ee683fa331336 Mon Sep 17 00:00:00 2001 From: mnunzio Date: Tue, 8 Mar 2022 18:04:08 +0100 Subject: [PATCH 02/17] feat: add support for async http library aiohttp --- pyodata/client.py | 72 +++++++++++++++++++++++++++++++--------- pyodata/v2/response.py | 32 ++++++++++++++++++ pyodata/v2/service.py | 74 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 153 insertions(+), 25 deletions(-) create mode 100644 pyodata/v2/response.py diff --git a/pyodata/client.py b/pyodata/client.py index d7fb9377..d958465e 100644 --- a/pyodata/client.py +++ b/pyodata/client.py @@ -6,6 +6,20 @@ import pyodata.v2.model import pyodata.v2.service from pyodata.exceptions import PyODataException, HttpError +from pyodata.v2.response import Response + + +async def _async_fetch_metadata(connection, url, logger): + logger.info('Fetching metadata') + + async with connection.get(url + '$metadata') as async_response: + resp = Response() + resp.url = async_response.url + resp.headers = async_response.headers + resp.status_code = async_response.status + resp.content = await async_response.read() + + return _common_fetch_metadata(resp, logger) def _fetch_metadata(connection, url, logger): @@ -13,6 +27,10 @@ def _fetch_metadata(connection, url, logger): logger.info('Fetching metadata') resp = connection.get(url + '$metadata') + return _common_fetch_metadata(resp, logger) + + +def _common_fetch_metadata(resp, logger): logger.debug('Retrieved the response:\n%s\n%s', '\n'.join((f'H: {key}: {value}' for key, value in resp.headers.items())), resp.content) @@ -37,6 +55,25 @@ class Client: ODATA_VERSION_2 = 2 + @staticmethod + async def build_async_client(url, connection, odata_version=ODATA_VERSION_2, namespaces=None, + config: pyodata.v2.model.Config = None, metadata: str = None): + """Create instance of the OData Client for given URL""" + + logger = logging.getLogger('pyodata.client') + + if odata_version == Client.ODATA_VERSION_2: + + # sanitize url + url = url.rstrip('/') + '/' + + if metadata is None: + metadata = await _async_fetch_metadata(connection, url, logger) + else: + logger.info('Using static metadata') + return Client._build_service(logger, url, connection, odata_version, namespaces, config, metadata) + raise PyODataException(f'No implementation for selected odata version {odata_version}') + def __new__(cls, url, connection, odata_version=ODATA_VERSION_2, namespaces=None, config: pyodata.v2.model.Config = None, metadata: str = None): """Create instance of the OData Client for given URL""" @@ -53,24 +90,29 @@ def __new__(cls, url, connection, odata_version=ODATA_VERSION_2, namespaces=None else: logger.info('Using static metadata') - if config is not None and namespaces is not None: - raise PyODataException('You cannot pass namespaces and config at the same time') + return Client._build_service(logger, url, connection, odata_version, namespaces, config, metadata) + raise PyODataException(f'No implementation for selected odata version {odata_version}') - if config is None: - config = pyodata.v2.model.Config() + @staticmethod + def _build_service(logger, url, connection, odata_version=ODATA_VERSION_2, namespaces=None, + config: pyodata.v2.model.Config = None, metadata: str = None): - if namespaces is not None: - warnings.warn("Passing namespaces directly is deprecated. Use class Config instead", DeprecationWarning) - config.namespaces = namespaces + if config is not None and namespaces is not None: + raise PyODataException('You cannot pass namespaces and config at the same time') - # create model instance from received metadata - logger.info('Creating OData Schema (version: %d)', odata_version) - schema = pyodata.v2.model.MetadataBuilder(metadata, config=config).build() + if config is None: + config = pyodata.v2.model.Config() - # create service instance based on model we have - logger.info('Creating OData Service (version: %d)', odata_version) - service = pyodata.v2.service.Service(url, schema, connection, config=config) + if namespaces is not None: + warnings.warn("Passing namespaces directly is deprecated. Use class Config instead", DeprecationWarning) + config.namespaces = namespaces - return service + # create model instance from received metadata + logger.info('Creating OData Schema (version: %d)', odata_version) + schema = pyodata.v2.model.MetadataBuilder(metadata, config=config).build() - raise PyODataException(f'No implementation for selected odata version {odata_version}') + # create service instance based on model we have + logger.info('Creating OData Service (version: %d)', odata_version) + service = pyodata.v2.service.Service(url, schema, connection, config=config) + + return service diff --git a/pyodata/v2/response.py b/pyodata/v2/response.py new file mode 100644 index 00000000..2dce7056 --- /dev/null +++ b/pyodata/v2/response.py @@ -0,0 +1,32 @@ +""" +Utility class to standardize response + +Author: Alberto Moio , Nunzio Mauro +Date: 2017-08-21 +""" +import json + + +class Response: + """Representation of http response in a standard form already used by handlers""" + + __attrs__ = [ + 'content', 'status_code', 'headers', 'url' + ] + + def __init__(self): + self.status_code = None + self.headers = None + self.url = None + self.content = None + + @property + def text(self): + """Textual representation of response content""" + + return self.content.decode('utf-8') + + def json(self): + """JSON representation of response content""" + + return json.loads(self.text) diff --git a/pyodata/v2/service.py b/pyodata/v2/service.py index f75f0d92..05dda32d 100644 --- a/pyodata/v2/service.py +++ b/pyodata/v2/service.py @@ -18,6 +18,7 @@ from pyodata.exceptions import HttpError, PyODataException, ExpressionError, ProgramError from . import model +from .response import Response LOGGER_NAME = 'pyodata.service' @@ -292,14 +293,7 @@ def add_headers(self, value): self._headers.update(value) - def execute(self): - """Fetches HTTP response and returns processed result - - Sends the query-request to the OData service, returning a client-side Enumerable for - subsequent in-memory operations. - - Fetches HTTP response and returns processed result""" - + def _build_request(self): if self._next_url: url = self._next_url else: @@ -315,10 +309,47 @@ def execute(self): if body: self._logger.debug(' body: %s', body) - params = urlencode(self.get_query_params()) + params = self.get_query_params() + + return url, body, headers, params + + async def async_execute(self): + """Fetches HTTP response and returns processed result + + Sends the query-request to the OData service, returning a client-side Enumerable for + subsequent in-memory operations. + + Fetches HTTP response and returns processed result""" + + url, body, headers, params = self._build_request() + async with self._connection.request(self.get_method(), + url, + headers=headers, + params=params, + data=body) as async_response: + response = Response() + response.url = async_response.url + response.headers = async_response.headers + response.status_code = async_response.status + response.content = await async_response.read() + return self._call_handler(response) + + def execute(self): + """Fetches HTTP response and returns processed result + + Sends the query-request to the OData service, returning a client-side Enumerable for + subsequent in-memory operations. + + Fetches HTTP response and returns processed result""" + + url, body, headers, params = self._build_request() + response = self._connection.request( - self.get_method(), url, headers=headers, params=params, data=body) + self.get_method(), url, headers=headers, params=urlencode(params), data=body) + return self._call_handler(response) + + def _call_handler(self, response): self._logger.debug('Received response') self._logger.debug(' url: %s', response.url) self._logger.debug(' headers: %s', response.headers) @@ -858,6 +889,19 @@ def __getattr__(self, attr): raise AttributeError('EntityType {0} does not have Property {1}: {2}' .format(self._entity_type.name, attr, str(ex))) + async def getattr(self, attr): + """Get cached value of attribute or do async call to service to recover attribute value""" + try: + return self._cache[attr] + except KeyError: + try: + value = await self.get_proprty(attr).async_execute() + self._cache[attr] = value + return value + except KeyError as ex: + raise AttributeError('EntityType {0} does not have Property {1}: {2}' + .format(self._entity_type.name, attr, str(ex))) + def nav(self, nav_property): """Navigates to given navigation property and returns the EntitySetProxy""" @@ -1686,6 +1730,16 @@ def http_get(self, path, connection=None): return conn.get(urljoin(self._url, path)) + async def async_http_get(self, path, connection=None): + """HTTP GET response for the passed path in the service""" + + conn = connection + if conn is None: + conn = self._connection + + async with conn.get(urljoin(self._url, path)) as resp: + return resp + def http_get_odata(self, path, handler, connection=None): """HTTP GET request proxy for the passed path in the service""" From ac550d843913e9f092ea4f6693e9947253f70909 Mon Sep 17 00:00:00 2001 From: Alberto Moio Date: Thu, 10 Mar 2022 23:22:58 +0100 Subject: [PATCH 03/17] Added: prefix async getattr, atom response --- pyodata/client.py | 10 ++++------ pyodata/v2/response.py | 32 -------------------------------- pyodata/v2/service.py | 15 +++++++-------- 3 files changed, 11 insertions(+), 46 deletions(-) delete mode 100644 pyodata/v2/response.py diff --git a/pyodata/client.py b/pyodata/client.py index d958465e..f2383fc8 100644 --- a/pyodata/client.py +++ b/pyodata/client.py @@ -6,18 +6,16 @@ import pyodata.v2.model import pyodata.v2.service from pyodata.exceptions import PyODataException, HttpError -from pyodata.v2.response import Response async def _async_fetch_metadata(connection, url, logger): logger.info('Fetching metadata') async with connection.get(url + '$metadata') as async_response: - resp = Response() - resp.url = async_response.url - resp.headers = async_response.headers - resp.status_code = async_response.status - resp.content = await async_response.read() + resp = pyodata.v2.service.ODataHttpResponse(url=async_response.url, + headers=async_response.headers, + status_code=async_response.status, + content=await async_response.read()) return _common_fetch_metadata(resp, logger) diff --git a/pyodata/v2/response.py b/pyodata/v2/response.py deleted file mode 100644 index 2dce7056..00000000 --- a/pyodata/v2/response.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -Utility class to standardize response - -Author: Alberto Moio , Nunzio Mauro -Date: 2017-08-21 -""" -import json - - -class Response: - """Representation of http response in a standard form already used by handlers""" - - __attrs__ = [ - 'content', 'status_code', 'headers', 'url' - ] - - def __init__(self): - self.status_code = None - self.headers = None - self.url = None - self.content = None - - @property - def text(self): - """Textual representation of response content""" - - return self.content.decode('utf-8') - - def json(self): - """JSON representation of response content""" - - return json.loads(self.text) diff --git a/pyodata/v2/service.py b/pyodata/v2/service.py index 05dda32d..453199c3 100644 --- a/pyodata/v2/service.py +++ b/pyodata/v2/service.py @@ -18,7 +18,6 @@ from pyodata.exceptions import HttpError, PyODataException, ExpressionError, ProgramError from . import model -from .response import Response LOGGER_NAME = 'pyodata.service' @@ -103,7 +102,8 @@ def decode(message): class ODataHttpResponse: """Representation of http response""" - def __init__(self, headers, status_code, content=None): + def __init__(self, headers, status_code, content=None, url=None): + self.url = url self.headers = headers self.status_code = status_code self.content = content @@ -327,11 +327,10 @@ async def async_execute(self): headers=headers, params=params, data=body) as async_response: - response = Response() - response.url = async_response.url - response.headers = async_response.headers - response.status_code = async_response.status - response.content = await async_response.read() + response = ODataHttpResponse(url=async_response.url, + headers=async_response.headers, + status_code=async_response.status, + content=await async_response.read()) return self._call_handler(response) def execute(self): @@ -889,7 +888,7 @@ def __getattr__(self, attr): raise AttributeError('EntityType {0} does not have Property {1}: {2}' .format(self._entity_type.name, attr, str(ex))) - async def getattr(self, attr): + async def async_getattr(self, attr): """Get cached value of attribute or do async call to service to recover attribute value""" try: return self._cache[attr] From 724bf708eb7979e244c118482b0cec97a74f06e0 Mon Sep 17 00:00:00 2001 From: Alberto Moio Date: Thu, 10 Mar 2022 23:23:09 +0100 Subject: [PATCH 04/17] added tests --- tests/conftest.py | 8 +++ tests/test_async_client.py | 135 +++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 tests/test_async_client.py diff --git a/tests/conftest.py b/tests/conftest.py index 4eb2d2c3..54f8c6a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """PyTest Fixtures""" +import asyncio import logging import os import pytest @@ -139,3 +140,10 @@ def type_date_time(): @pytest.fixture def type_date_time_offset(): return Types.from_name('Edm.DateTimeOffset') + + +@pytest.fixture +def event_loop(): + loop = asyncio.new_event_loop() + yield loop + loop.run_until_complete(asyncio.sleep(0.1, loop=loop)) diff --git a/tests/test_async_client.py b/tests/test_async_client.py new file mode 100644 index 00000000..d86fd7bb --- /dev/null +++ b/tests/test_async_client.py @@ -0,0 +1,135 @@ +"""PyOData Client tests""" +from unittest.mock import patch, AsyncMock + +import aiohttp +import pytest +import requests.adapters + +import pyodata.v2.service +from pyodata import Client +from pyodata.exceptions import PyODataException, HttpError +from pyodata.v2.model import ParserError, PolicyWarning, PolicyFatal, PolicyIgnore, Config + +SERVICE_URL = 'http://example.com' + + +@pytest.mark.asyncio +async def test_invalid_odata_version(): + """Check handling of request for invalid OData version implementation""" + + with pytest.raises(PyODataException) as e_info: + async with aiohttp.ClientSession() as client: + await Client.build_async_client(SERVICE_URL, client, 'INVALID VERSION') + + assert str(e_info.value).startswith('No implementation for selected odata version') + + +@pytest.mark.asyncio +async def test_create_client_for_local_metadata(metadata): + """Check client creation for valid use case with local metadata""" + + async with aiohttp.ClientSession() as client: + service_client = await Client.build_async_client(SERVICE_URL, client, metadata=metadata) + + assert isinstance(service_client, pyodata.v2.service.Service) + assert service_client.schema.is_valid == True + + assert len(service_client.schema.entity_sets) != 0 + + +@patch("pyodata.client._async_fetch_metadata") +@pytest.mark.parametrize("content_type", ['application/xml', 'application/atom+xml', 'text/xml']) +@pytest.mark.asyncio +async def test_create_service_application(mock_fetch_metadata, metadata, content_type): + """Check client creation for valid MIME types""" + mock_fetch_metadata.return_value = metadata + + async with aiohttp.ClientSession() as client: + service_client = await Client.build_async_client(SERVICE_URL, client) + + assert isinstance(service_client, pyodata.v2.service.Service) + + # one more test for '/' terminated url + + service_client = await Client.build_async_client(SERVICE_URL + '/', requests) + + assert isinstance(service_client, pyodata.v2.service.Service) + assert service_client.schema.is_valid + + +@patch("aiohttp.client.ClientSession.get") +@pytest.mark.asyncio +async def test_metadata_not_reachable(mock): + """Check handling of not reachable service metadata""" + + response = AsyncMock() + response.status = 404 + response.headers = {'content-type': 'text/html'} + mock.return_value.__aenter__.return_value = response + + with pytest.raises(HttpError) as e_info: + async with aiohttp.ClientSession() as client: + await Client.build_async_client(SERVICE_URL, client) + + assert str(e_info.value).startswith('Metadata request failed') + + +@patch("aiohttp.client.ClientSession.get") +@pytest.mark.asyncio +async def test_metadata_saml_not_authorized(mock): + """Check handling of not SAML / OAuth unauthorized response""" + + response = AsyncMock() + response.status = 200 + response.headers = {'content-type': 'text/html; charset=utf-8'} + mock.return_value.__aenter__.return_value = response + + with pytest.raises(HttpError) as e_info: + async with aiohttp.ClientSession() as client: + await Client.build_async_client(SERVICE_URL, client) + + assert str(e_info.value).startswith('Metadata request did not return XML, MIME type:') + + +@patch("pyodata.client._async_fetch_metadata") +@patch('warnings.warn') +@pytest.mark.asyncio +async def test_client_custom_configuration(mock_warning, mock_fetch_metadata, metadata): + """Check client creation for custom configuration""" + + mock_fetch_metadata.return_value = metadata + + namespaces = { + 'edmx': "customEdmxUrl.com", + 'edm': 'customEdmUrl.com' + } + + custom_config = Config( + xml_namespaces=namespaces, + default_error_policy=PolicyFatal(), + custom_error_policies={ + ParserError.ANNOTATION: PolicyWarning(), + ParserError.ASSOCIATION: PolicyIgnore() + }) + + with pytest.raises(PyODataException) as e_info: + async with aiohttp.ClientSession() as client: + await Client.build_async_client(SERVICE_URL, client, config=custom_config, namespaces=namespaces) + + assert str(e_info.value) == 'You cannot pass namespaces and config at the same time' + + async with aiohttp.ClientSession() as client: + service = await Client.build_async_client(SERVICE_URL, client, namespaces=namespaces) + + mock_warning.assert_called_with( + 'Passing namespaces directly is deprecated. Use class Config instead', + DeprecationWarning + ) + assert isinstance(service, pyodata.v2.service.Service) + assert service.schema.config.namespaces == namespaces + + async with aiohttp.ClientSession() as client: + service = await Client.build_async_client(SERVICE_URL, client, config=custom_config) + + assert isinstance(service, pyodata.v2.service.Service) + assert service.schema.config == custom_config From bf18c7343cc47ec3b6b406b8ee49370b31da71b8 Mon Sep 17 00:00:00 2001 From: Alberto Moio Date: Fri, 11 Mar 2022 18:41:49 +0100 Subject: [PATCH 05/17] updated dev-requirements.txt --- dev-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 77e85c8b..2afdef75 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,6 @@ requests==2.23.0 +pytest-asyncio == 0.15.1 +aiohttp==3.8.1 pytest>=4.6.0 pytest-aiohttp>=1.0.4 responses>=0.8.1 From 1966c2630ce01965d90950c79c57cb894c0d7a50 Mon Sep 17 00:00:00 2001 From: mnunzio Date: Mon, 14 Mar 2022 18:33:07 +0100 Subject: [PATCH 06/17] feat: update async_client tests rewrite tests using pytest-aiohttp --- tests/test_async_client.py | 82 ++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/tests/test_async_client.py b/tests/test_async_client.py index d86fd7bb..bf4cbdac 100644 --- a/tests/test_async_client.py +++ b/tests/test_async_client.py @@ -3,17 +3,16 @@ import aiohttp import pytest -import requests.adapters +from aiohttp import web import pyodata.v2.service from pyodata import Client from pyodata.exceptions import PyODataException, HttpError from pyodata.v2.model import ParserError, PolicyWarning, PolicyFatal, PolicyIgnore, Config -SERVICE_URL = 'http://example.com' +SERVICE_URL = '' -@pytest.mark.asyncio async def test_invalid_odata_version(): """Check handling of request for invalid OData version implementation""" @@ -24,7 +23,6 @@ async def test_invalid_odata_version(): assert str(e_info.value).startswith('No implementation for selected odata version') -@pytest.mark.asyncio async def test_create_client_for_local_metadata(metadata): """Check client creation for valid use case with local metadata""" @@ -37,68 +35,63 @@ async def test_create_client_for_local_metadata(metadata): assert len(service_client.schema.entity_sets) != 0 -@patch("pyodata.client._async_fetch_metadata") +def generate_metadata_response(headers=None, body=None, status=200): + async def metadata_repsonse(request): + return web.Response(status=status, headers=headers, body=body) + + return metadata_repsonse + + @pytest.mark.parametrize("content_type", ['application/xml', 'application/atom+xml', 'text/xml']) -@pytest.mark.asyncio -async def test_create_service_application(mock_fetch_metadata, metadata, content_type): +async def test_create_service_application(aiohttp_client, metadata, content_type): """Check client creation for valid MIME types""" - mock_fetch_metadata.return_value = metadata - async with aiohttp.ClientSession() as client: - service_client = await Client.build_async_client(SERVICE_URL, client) + app = web.Application() + app.router.add_get('/$metadata', generate_metadata_response(headers={'content-type': content_type}, body=metadata)) + client = await aiohttp_client(app) - assert isinstance(service_client, pyodata.v2.service.Service) + service_client = await Client.build_async_client(SERVICE_URL, client) - # one more test for '/' terminated url + assert isinstance(service_client, pyodata.v2.service.Service) - service_client = await Client.build_async_client(SERVICE_URL + '/', requests) + # one more test for '/' terminated url - assert isinstance(service_client, pyodata.v2.service.Service) - assert service_client.schema.is_valid + service_client = await Client.build_async_client(SERVICE_URL + '/', client) + + assert isinstance(service_client, pyodata.v2.service.Service) + assert service_client.schema.is_valid -@patch("aiohttp.client.ClientSession.get") -@pytest.mark.asyncio -async def test_metadata_not_reachable(mock): +async def test_metadata_not_reachable(aiohttp_client): """Check handling of not reachable service metadata""" - response = AsyncMock() - response.status = 404 - response.headers = {'content-type': 'text/html'} - mock.return_value.__aenter__.return_value = response + app = web.Application() + app.router.add_get('/$metadata', generate_metadata_response(headers={'content-type': 'text/html'}, status=404)) + client = await aiohttp_client(app) with pytest.raises(HttpError) as e_info: - async with aiohttp.ClientSession() as client: - await Client.build_async_client(SERVICE_URL, client) + await Client.build_async_client(SERVICE_URL, client) assert str(e_info.value).startswith('Metadata request failed') -@patch("aiohttp.client.ClientSession.get") -@pytest.mark.asyncio -async def test_metadata_saml_not_authorized(mock): +async def test_metadata_saml_not_authorized(aiohttp_client): """Check handling of not SAML / OAuth unauthorized response""" - response = AsyncMock() - response.status = 200 - response.headers = {'content-type': 'text/html; charset=utf-8'} - mock.return_value.__aenter__.return_value = response + app = web.Application() + app.router.add_get('/$metadata', generate_metadata_response(headers={'content-type': 'text/html; charset=utf-8'})) + client = await aiohttp_client(app) with pytest.raises(HttpError) as e_info: - async with aiohttp.ClientSession() as client: - await Client.build_async_client(SERVICE_URL, client) + await Client.build_async_client(SERVICE_URL, client) assert str(e_info.value).startswith('Metadata request did not return XML, MIME type:') -@patch("pyodata.client._async_fetch_metadata") @patch('warnings.warn') -@pytest.mark.asyncio -async def test_client_custom_configuration(mock_warning, mock_fetch_metadata, metadata): +async def test_client_custom_configuration(mock_warning, aiohttp_client, metadata): """Check client creation for custom configuration""" - mock_fetch_metadata.return_value = metadata - namespaces = { 'edmx': "customEdmxUrl.com", 'edm': 'customEdmUrl.com' @@ -112,14 +105,16 @@ async def test_client_custom_configuration(mock_warning, mock_fetch_metadata, me ParserError.ASSOCIATION: PolicyIgnore() }) + app = web.Application() + app.router.add_get('/$metadata', generate_metadata_response(headers={'content-type': 'application/xml'},body=metadata)) + client = await aiohttp_client(app) + with pytest.raises(PyODataException) as e_info: - async with aiohttp.ClientSession() as client: - await Client.build_async_client(SERVICE_URL, client, config=custom_config, namespaces=namespaces) + await Client.build_async_client(SERVICE_URL, client, config=custom_config, namespaces=namespaces) assert str(e_info.value) == 'You cannot pass namespaces and config at the same time' - async with aiohttp.ClientSession() as client: - service = await Client.build_async_client(SERVICE_URL, client, namespaces=namespaces) + service = await Client.build_async_client(SERVICE_URL, client, namespaces=namespaces) mock_warning.assert_called_with( 'Passing namespaces directly is deprecated. Use class Config instead', @@ -128,8 +123,7 @@ async def test_client_custom_configuration(mock_warning, mock_fetch_metadata, me assert isinstance(service, pyodata.v2.service.Service) assert service.schema.config.namespaces == namespaces - async with aiohttp.ClientSession() as client: - service = await Client.build_async_client(SERVICE_URL, client, config=custom_config) + service = await Client.build_async_client(SERVICE_URL, client, config=custom_config) assert isinstance(service, pyodata.v2.service.Service) assert service.schema.config == custom_config From 5998c03658ec583d59b8e1c9ef293e571c179a90 Mon Sep 17 00:00:00 2001 From: mnunzio Date: Mon, 14 Mar 2022 22:04:24 +0100 Subject: [PATCH 07/17] feat: update dev-requirements.txt add pytest-aiohttp remove pytest-asyncio remove aiohttp --- dev-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 2afdef75..593dc17a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,5 @@ requests==2.23.0 -pytest-asyncio == 0.15.1 -aiohttp==3.8.1 +pytest-aiohttp == 1.0.4 pytest>=4.6.0 pytest-aiohttp>=1.0.4 responses>=0.8.1 From b90e362c57d406ce2ed174e32747061f7e93a2e3 Mon Sep 17 00:00:00 2001 From: mnunzio Date: Mon, 14 Mar 2022 22:38:23 +0100 Subject: [PATCH 08/17] feat: clean useless code --- tests/conftest.py | 10 ++-------- tests/test_async_client.py | 5 +++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 54f8c6a1..d33b3874 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,9 @@ """PyTest Fixtures""" -import asyncio import logging import os + import pytest + from pyodata.v2.model import schema_from_xml, Types @@ -140,10 +141,3 @@ def type_date_time(): @pytest.fixture def type_date_time_offset(): return Types.from_name('Edm.DateTimeOffset') - - -@pytest.fixture -def event_loop(): - loop = asyncio.new_event_loop() - yield loop - loop.run_until_complete(asyncio.sleep(0.1, loop=loop)) diff --git a/tests/test_async_client.py b/tests/test_async_client.py index bf4cbdac..ffbfa4fb 100644 --- a/tests/test_async_client.py +++ b/tests/test_async_client.py @@ -1,5 +1,5 @@ """PyOData Client tests""" -from unittest.mock import patch, AsyncMock +from unittest.mock import patch import aiohttp import pytest @@ -106,7 +106,8 @@ async def test_client_custom_configuration(mock_warning, aiohttp_client, metadat }) app = web.Application() - app.router.add_get('/$metadata', generate_metadata_response(headers={'content-type': 'application/xml'},body=metadata)) + app.router.add_get('/$metadata', + generate_metadata_response(headers={'content-type': 'application/xml'}, body=metadata)) client = await aiohttp_client(app) with pytest.raises(PyODataException) as e_info: From 4fe4d9e46af649302b95f1e1c24d87c3119b0283 Mon Sep 17 00:00:00 2001 From: I335851 Date: Thu, 11 Aug 2022 16:24:07 +0200 Subject: [PATCH 09/17] chore: fix dev-requirements duplicate after merge --- dev-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 07862bd9..4aceb8b3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,6 @@ requests>=2.28.1 pytest>=7.1.2 pytest-aiohttp>=1.0.4 responses>=0.21.0 -responses>=0.21.0 setuptools>=38.2.4 setuptools-scm>=1.15.6 pylint==2.8.3 From 3f4b950483dba4d8cff6b0e4f4593447350ed35c Mon Sep 17 00:00:00 2001 From: I335851 Date: Thu, 11 Aug 2022 17:16:01 +0200 Subject: [PATCH 10/17] tests: move client tests to integration/networking_libraries --- tests/integration/__init__.py | 0 .../integration/networking_libraries/__init__.py | 0 .../networking_libraries/test_aiohttp_client.py} | 16 +++++++++++----- .../test_requests_client.py} | 6 +++++- 4 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/networking_libraries/__init__.py rename tests/{test_async_client.py => integration/networking_libraries/test_aiohttp_client.py} (94%) rename tests/{test_client.py => integration/networking_libraries/test_requests_client.py} (97%) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/networking_libraries/__init__.py b/tests/integration/networking_libraries/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_async_client.py b/tests/integration/networking_libraries/test_aiohttp_client.py similarity index 94% rename from tests/test_async_client.py rename to tests/integration/networking_libraries/test_aiohttp_client.py index ffbfa4fb..5310839d 100644 --- a/tests/test_async_client.py +++ b/tests/integration/networking_libraries/test_aiohttp_client.py @@ -1,4 +1,7 @@ -"""PyOData Client tests""" +""" Test the pyodata integration with aiohttp client, based on asyncio + +https://docs.aiohttp.org/en/stable/ +""" from unittest.mock import patch import aiohttp @@ -12,7 +15,7 @@ SERVICE_URL = '' - +@pytest.mark.asyncio async def test_invalid_odata_version(): """Check handling of request for invalid OData version implementation""" @@ -22,7 +25,7 @@ async def test_invalid_odata_version(): assert str(e_info.value).startswith('No implementation for selected odata version') - +@pytest.mark.asyncio async def test_create_client_for_local_metadata(metadata): """Check client creation for valid use case with local metadata""" @@ -34,7 +37,7 @@ async def test_create_client_for_local_metadata(metadata): assert len(service_client.schema.entity_sets) != 0 - +@pytest.mark.asyncio def generate_metadata_response(headers=None, body=None, status=200): async def metadata_repsonse(request): return web.Response(status=status, headers=headers, body=body) @@ -43,6 +46,7 @@ async def metadata_repsonse(request): @pytest.mark.parametrize("content_type", ['application/xml', 'application/atom+xml', 'text/xml']) +@pytest.mark.asyncio async def test_create_service_application(aiohttp_client, metadata, content_type): """Check client creation for valid MIME types""" @@ -62,6 +66,7 @@ async def test_create_service_application(aiohttp_client, metadata, content_type assert service_client.schema.is_valid +@pytest.mark.asyncio async def test_metadata_not_reachable(aiohttp_client): """Check handling of not reachable service metadata""" @@ -74,7 +79,7 @@ async def test_metadata_not_reachable(aiohttp_client): assert str(e_info.value).startswith('Metadata request failed') - +@pytest.mark.asyncio async def test_metadata_saml_not_authorized(aiohttp_client): """Check handling of not SAML / OAuth unauthorized response""" @@ -89,6 +94,7 @@ async def test_metadata_saml_not_authorized(aiohttp_client): @patch('warnings.warn') +@pytest.mark.asyncio async def test_client_custom_configuration(mock_warning, aiohttp_client, metadata): """Check client creation for custom configuration""" diff --git a/tests/test_client.py b/tests/integration/networking_libraries/test_requests_client.py similarity index 97% rename from tests/test_client.py rename to tests/integration/networking_libraries/test_requests_client.py index 53d8c937..e651e3a3 100644 --- a/tests/test_client.py +++ b/tests/integration/networking_libraries/test_requests_client.py @@ -1,4 +1,7 @@ -"""PyOData Client tests""" +""" Test the pyodata integration with Requests library. + +https://requests.readthedocs.io/en/latest/ +""" import responses import requests @@ -12,6 +15,7 @@ SERVICE_URL = 'http://example.com' + @responses.activate def test_invalid_odata_version(): """Check handling of request for invalid OData version implementation""" From 43990505accb423d7051f857bf9b429f88f0a051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Han=C3=A1k?= <47626173+phanak-sap@users.noreply.github.com> Date: Sun, 14 Aug 2022 16:04:37 +0200 Subject: [PATCH 11/17] ci: set fail-fast: false so all matrix jobs runs --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5470802d..2288257a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: - fail-fast: true + fail-fast: false matrix: python-version: ["3.7", "3.8", "3.9", "3.10.1"] lxml-version: ["4.1.1", "4.2.6", "4.3.5", "4.4.3", "4.5.2", "4.6.5", "4.7.1", "4.8.0", "4.9.1"] From 8bc2c69d0a16f2ea63274d9e6d0c7c5a90b0229c Mon Sep 17 00:00:00 2001 From: I335851 Date: Sun, 14 Aug 2022 18:51:22 +0200 Subject: [PATCH 12/17] tests: aiohttp - switch from unittest to pytest warnings Switch "from unittest.mock import patch" to pytest.warns. Maybe it is the reason why aiohttp and python 3.7 does not go together on CI builds, and if it is not a root cause, it looks cleaner anyway. Stick to one testing framework stuff. --- .../networking_libraries/test_aiohttp_client.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integration/networking_libraries/test_aiohttp_client.py b/tests/integration/networking_libraries/test_aiohttp_client.py index 5310839d..6906a68c 100644 --- a/tests/integration/networking_libraries/test_aiohttp_client.py +++ b/tests/integration/networking_libraries/test_aiohttp_client.py @@ -2,11 +2,9 @@ https://docs.aiohttp.org/en/stable/ """ -from unittest.mock import patch - import aiohttp -import pytest from aiohttp import web +import pytest import pyodata.v2.service from pyodata import Client @@ -93,9 +91,8 @@ async def test_metadata_saml_not_authorized(aiohttp_client): assert str(e_info.value).startswith('Metadata request did not return XML, MIME type:') -@patch('warnings.warn') @pytest.mark.asyncio -async def test_client_custom_configuration(mock_warning, aiohttp_client, metadata): +async def test_client_custom_configuration(aiohttp_client, metadata): """Check client creation for custom configuration""" namespaces = { @@ -121,12 +118,9 @@ async def test_client_custom_configuration(mock_warning, aiohttp_client, metadat assert str(e_info.value) == 'You cannot pass namespaces and config at the same time' - service = await Client.build_async_client(SERVICE_URL, client, namespaces=namespaces) + with pytest.warns(DeprecationWarning,match='Passing namespaces directly is deprecated. Use class Config instead'): + service = await Client.build_async_client(SERVICE_URL, client, namespaces=namespaces) - mock_warning.assert_called_with( - 'Passing namespaces directly is deprecated. Use class Config instead', - DeprecationWarning - ) assert isinstance(service, pyodata.v2.service.Service) assert service.schema.config.namespaces == namespaces From 917de927787bca0102a0b7029fbe2bb3fcc5ef71 Mon Sep 17 00:00:00 2001 From: I335851 Date: Sun, 14 Aug 2022 20:07:55 +0200 Subject: [PATCH 13/17] tests: requests - switch from unittest to pytest warnings Switch "from unittest.mock import patch" to pytest.warns. --- .../networking_libraries/test_requests_client.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/integration/networking_libraries/test_requests_client.py b/tests/integration/networking_libraries/test_requests_client.py index e651e3a3..6164861b 100644 --- a/tests/integration/networking_libraries/test_requests_client.py +++ b/tests/integration/networking_libraries/test_requests_client.py @@ -8,7 +8,6 @@ import pytest import pyodata import pyodata.v2.service -from unittest.mock import patch from pyodata.exceptions import PyODataException, HttpError from pyodata.v2.model import ParserError, PolicyWarning, PolicyFatal, PolicyIgnore, Config @@ -94,8 +93,7 @@ def test_metadata_saml_not_authorized(): @responses.activate -@patch('warnings.warn') -def test_client_custom_configuration(mock_warning, metadata): +def test_client_custom_configuration(metadata): """Check client creation for custom configuration""" responses.add( @@ -123,12 +121,9 @@ def test_client_custom_configuration(mock_warning, metadata): assert str(e_info.value) == 'You cannot pass namespaces and config at the same time' - client = pyodata.Client(SERVICE_URL, requests, namespaces=namespaces) + with pytest.warns(DeprecationWarning,match='Passing namespaces directly is deprecated. Use class Config instead'): + client = pyodata.Client(SERVICE_URL, requests, namespaces=namespaces) - mock_warning.assert_called_with( - 'Passing namespaces directly is deprecated. Use class Config instead', - DeprecationWarning - ) assert isinstance(client, pyodata.v2.service.Service) assert client.schema.config.namespaces == namespaces From 771312c093f64d85c739e81fb449b1d590dbfc53 Mon Sep 17 00:00:00 2001 From: I335851 Date: Sun, 14 Aug 2022 21:59:20 +0200 Subject: [PATCH 14/17] tests: add first httpx sync client tests --- dev-requirements.txt | 2 + .../test_httpx_client_sync.py | 61 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 tests/integration/networking_libraries/test_httpx_client_sync.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 4aceb8b3..9e4abbc1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,6 +2,8 @@ requests>=2.28.1 pytest>=7.1.2 pytest-aiohttp>=1.0.4 responses>=0.21.0 +httpx>=0.23.0 +respx>=0.19.2 setuptools>=38.2.4 setuptools-scm>=1.15.6 pylint==2.8.3 diff --git a/tests/integration/networking_libraries/test_httpx_client_sync.py b/tests/integration/networking_libraries/test_httpx_client_sync.py new file mode 100644 index 00000000..66f4f5b7 --- /dev/null +++ b/tests/integration/networking_libraries/test_httpx_client_sync.py @@ -0,0 +1,61 @@ +""" Test the pyodata integration with httpx client + +- it provided sync, requests like interface - FOCUS OF THIS TEST MODULE +- it provides asyncio interface as well + +https://www.python-httpx.org/ +""" + +import httpx +from httpx import Response +import respx +import pytest + +import pyodata.v2.service +from pyodata import Client +from pyodata.exceptions import PyODataException, HttpError +from pyodata.v2.model import ParserError, PolicyWarning, PolicyFatal, PolicyIgnore, Config + +SERVICE_URL = 'http://example.com' + +def test_invalid_odata_version(): + """Check handling of request for invalid OData version implementation""" + + with pytest.raises(PyODataException) as e_info: + pyodata.Client(SERVICE_URL, httpx, 'INVALID VERSION') + + assert str(e_info.value).startswith('No implementation for selected odata version') + + +def test_create_client_for_local_metadata(metadata): + """Check client creation for valid use case with local metadata""" + + client = pyodata.Client(SERVICE_URL, httpx, metadata=metadata) + + assert isinstance(client, pyodata.v2.service.Service) + assert client.schema.is_valid == True + assert len(client.schema.entity_sets) != 0 + + +@pytest.mark.parametrize("content_type", ['application/xml', 'application/atom+xml', 'text/xml']) +def test_create_service_application(respx_mock, metadata, content_type): + """Check client creation for valid MIME types""" + + headers = httpx.Headers( + {'Content-Type': content_type} + ) + + respx_mock.get(f"{SERVICE_URL}/$metadata").mock( + return_value=Response(status_code=200, + content=metadata, + headers=headers, + ) + ) + + client = pyodata.Client(SERVICE_URL, httpx) + assert isinstance(client, pyodata.v2.service.Service) + + # one more test for '/' terminated url + client = pyodata.Client(SERVICE_URL + '/', httpx) + assert isinstance(client, pyodata.v2.service.Service) + assert client.schema.is_valid From 5c40988d863ff8bbfbc826432e7f69de03c0c4d1 Mon Sep 17 00:00:00 2001 From: I335851 Date: Tue, 16 Aug 2022 12:53:03 +0200 Subject: [PATCH 15/17] tests: rest of sync httpx client tests --- .../test_httpx_client_sync.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/integration/networking_libraries/test_httpx_client_sync.py b/tests/integration/networking_libraries/test_httpx_client_sync.py index 66f4f5b7..dcc87378 100644 --- a/tests/integration/networking_libraries/test_httpx_client_sync.py +++ b/tests/integration/networking_libraries/test_httpx_client_sync.py @@ -41,6 +41,7 @@ def test_create_client_for_local_metadata(metadata): def test_create_service_application(respx_mock, metadata, content_type): """Check client creation for valid MIME types""" + # Note: respx_mock is provided by respx package as pytest helper headers = httpx.Headers( {'Content-Type': content_type} ) @@ -59,3 +60,85 @@ def test_create_service_application(respx_mock, metadata, content_type): client = pyodata.Client(SERVICE_URL + '/', httpx) assert isinstance(client, pyodata.v2.service.Service) assert client.schema.is_valid + + +def test_metadata_not_reachable(respx_mock): + """Check handling of not reachable service metadata""" + + headers = httpx.Headers( + {'Content-Type': 'text/html'} + ) + + respx_mock.get(f"{SERVICE_URL}/$metadata").mock( + return_value=Response(status_code=404, + headers=headers, + ) + ) + + with pytest.raises(HttpError) as e_info: + pyodata.Client(SERVICE_URL, httpx) + + assert str(e_info.value).startswith('Metadata request failed') + + +def test_metadata_saml_not_authorized(respx_mock): + """Check handling of not SAML / OAuth unauthorized response""" + + headers = httpx.Headers( + {'Content-Type': 'text/html; charset=utf-8'} + ) + + respx_mock.get(f"{SERVICE_URL}/$metadata").mock( + return_value=Response(status_code=200, + headers=headers, + ) + ) + + with pytest.raises(HttpError) as e_info: + pyodata.Client(SERVICE_URL, httpx) + + assert str(e_info.value).startswith('Metadata request did not return XML, MIME type:') + + +def test_client_custom_configuration(respx_mock,metadata): + """Check client creation for custom configuration""" + + headers = httpx.Headers( + {'Content-Type': 'application/xml'} + ) + + respx_mock.get(f"{SERVICE_URL}/$metadata").mock( + return_value=Response(status_code=200, + headers=headers, + content=metadata, + ) + ) + + namespaces = { + 'edmx': "customEdmxUrl.com", + 'edm': 'customEdmUrl.com' + } + + custom_config = Config( + xml_namespaces=namespaces, + default_error_policy=PolicyFatal(), + custom_error_policies={ + ParserError.ANNOTATION: PolicyWarning(), + ParserError.ASSOCIATION: PolicyIgnore() + }) + + with pytest.raises(PyODataException) as e_info: + client = pyodata.Client(SERVICE_URL, httpx, config=custom_config, namespaces=namespaces) + + assert str(e_info.value) == 'You cannot pass namespaces and config at the same time' + + with pytest.warns(DeprecationWarning,match='Passing namespaces directly is deprecated. Use class Config instead'): + client = pyodata.Client(SERVICE_URL, httpx, namespaces=namespaces) + + assert isinstance(client, pyodata.v2.service.Service) + assert client.schema.config.namespaces == namespaces + + client = pyodata.Client(SERVICE_URL, httpx, config=custom_config) + + assert isinstance(client, pyodata.v2.service.Service) + assert client.schema.config == custom_config \ No newline at end of file From 718635cc095402afe90915c83ef3f5b4124c3b5c Mon Sep 17 00:00:00 2001 From: I335851 Date: Thu, 18 Aug 2022 21:45:16 +0200 Subject: [PATCH 16/17] tests: fix small typo --- .../integration/networking_libraries/test_aiohttp_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/networking_libraries/test_aiohttp_client.py b/tests/integration/networking_libraries/test_aiohttp_client.py index 6906a68c..fff0d28d 100644 --- a/tests/integration/networking_libraries/test_aiohttp_client.py +++ b/tests/integration/networking_libraries/test_aiohttp_client.py @@ -37,10 +37,11 @@ async def test_create_client_for_local_metadata(metadata): @pytest.mark.asyncio def generate_metadata_response(headers=None, body=None, status=200): - async def metadata_repsonse(request): + + async def metadata_response(request): return web.Response(status=status, headers=headers, body=body) - return metadata_repsonse + return metadata_response @pytest.mark.parametrize("content_type", ['application/xml', 'application/atom+xml', 'text/xml']) From f72fd36f61b6b943b1e62498bd6c3e01e45a5781 Mon Sep 17 00:00:00 2001 From: I335851 Date: Sat, 27 Aug 2022 12:51:14 +0200 Subject: [PATCH 17/17] tests: rm unecessary init.py (autogenerated) --- tests/integration/__init__.py | 0 tests/integration/networking_libraries/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/integration/__init__.py delete mode 100644 tests/integration/networking_libraries/__init__.py diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/integration/networking_libraries/__init__.py b/tests/integration/networking_libraries/__init__.py deleted file mode 100644 index e69de29b..00000000