From 4f3984ca496ce73259393195417ca889c8c354a0 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Wed, 19 Oct 2022 16:18:47 -0700 Subject: [PATCH] fix ci for core (#26923) * fix ci for core * updates * updates * updates * updates * updates * adding additional version override support * update * update * update * update conda test-requirements to include azure-mgmt-storage that was previously coming dependency tools/azure-sdk-tools Co-authored-by: scbedd <45376673+scbedd@users.noreply.github.com> --- eng/conda_test_requirements.txt | 1 + eng/tox/install_depend_packages.py | 54 ++++++++++++++++++- .../dev_requirements.txt | 3 +- .../async_tests/test_base_polling_async.py | 2 - .../tests/async_tests/test_polling_async.py | 8 +-- sdk/core/azure-core/tests/test_polling.py | 8 +-- .../asynctests/test_async_arm_polling.py | 2 - .../azure-mgmt-core/tests/test_arm_polling.py | 2 - .../azure-identity/dev_requirements.txt | 3 +- .../dev_requirements.txt | 1 + .../dev_requirements.txt | 1 + .../azure-keyvault-keys/dev_requirements.txt | 1 + .../dev_requirements.txt | 1 + .../dev_requirements.txt | 3 +- .../azure-storage-blob/dev_requirements.txt | 2 +- .../azure-data-tables/dev_requirements.txt | 1 + 16 files changed, 73 insertions(+), 20 deletions(-) diff --git a/eng/conda_test_requirements.txt b/eng/conda_test_requirements.txt index 45c8aeea6380d..c3d01d4b10757 100644 --- a/eng/conda_test_requirements.txt +++ b/eng/conda_test_requirements.txt @@ -12,3 +12,4 @@ adal setuptools==46.4.0 pytest-asyncio==0.12.0 -e sdk/core/azure-core/tests/testserver_tests/coretestserver +azure-mgmt-storage \ No newline at end of file diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 638f7e19c8741..ed6f0c4a1e226 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -12,11 +12,18 @@ import logging import re from subprocess import check_call +from typing import TYPE_CHECKING from pkg_resources import parse_version from pypi_tools.pypi import PyPIClient +from packaging.specifiers import SpecifierSet +from packaging.version import Version, parse +import pdb from ci_tools.parsing import ParsedSetup, parse_require + +from typing import List + DEV_REQ_FILE = "dev_requirements.txt" NEW_DEV_REQ_FILE = "new_dev_requirements.txt" PKGS_TXT_FILE = "packages.txt" @@ -33,11 +40,19 @@ "azure-core": "1.11.0", "requests": "2.19.0", "six": "1.12.0", - "cryptography": "3.3.2" + "cryptography": "3.3.2", } MAXIMUM_VERSION_SUPPORTED_OVERRIDE = {"cryptography": "4.0.0"} +SPECIAL_CASE_OVERRIDES = { + # this package has an override + "azure-core": { + # if the version being installed matches this specifier, add the listed packages to the install list + "<1.24.0": ["msrest<0.7.0"] + } +} + def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): # This method identifies latest/ minimal version of dependent packages and installs them from pyPI @@ -46,13 +61,25 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): # Minimum type will find minimum version on PyPI that satisfies requires of given package name released_packages = find_released_packages(setup_py_file_path, dependency_type) + + override_added_packages = [] + + # new section added to account for difficulties with msrest + for pkg_spec in released_packages: + override_added_packages.extend(check_pkg_against_overrides(pkg_spec)) + logging.info("%s released packages: %s", dependency_type, released_packages) # filter released packages from dev_requirements and create a new file "new_dev_requirements.txt" dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir) + if override_added_packages: + logging.info(f"Expanding the requirement set by the packages {override_added_packages}.") + + install_set = released_packages + list(set(override_added_packages)) + # install released dependent packages if released_packages or dev_req_file_path: - install_packages(released_packages, dev_req_file_path) + install_packages(install_set, dev_req_file_path) if released_packages: # create a file with list of packages and versions found based on minimum or latest check on PyPI @@ -64,6 +91,28 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): logging.info("Created file %s to track azure packages found on PyPI", pkgs_file_path) +def check_pkg_against_overrides(pkg_specifier: str) -> List[str]: + """ + Checks a set of package specifiers of form "[A==1.0.0, B=2.0.0]". Used to inject additional package installations + as indicated by the SPECIAL_CASE_OVERRIDES dictionary. + + :param str pkg_specifier: A specifically targeted package that is about to be passed to install_packages. + """ + additional_installs = [] + target_package, target_version = pkg_specifier.split("==") + + target_version = Version(target_version) + if target_package in SPECIAL_CASE_OVERRIDES: + special_case_specifiers = SPECIAL_CASE_OVERRIDES[target_package] + + for specifier_set in special_case_specifiers.keys(): + spec = SpecifierSet(specifier_set) + if target_version in spec: + additional_installs.extend(special_case_specifiers[specifier_set]) + + return additional_installs + + def find_released_packages(setup_py_path, dependency_type): # this method returns list of required available package on PyPI in format == @@ -72,6 +121,7 @@ def find_released_packages(setup_py_path, dependency_type): # Get available version on PyPI for each required package avlble_packages = [x for x in map(lambda x: process_requirement(x, dependency_type), requires) if x] + return avlble_packages diff --git a/sdk/communication/azure-communication-identity/dev_requirements.txt b/sdk/communication/azure-communication-identity/dev_requirements.txt index 9733800699af6..dd8edf9aa0ab3 100644 --- a/sdk/communication/azure-communication-identity/dev_requirements.txt +++ b/sdk/communication/azure-communication-identity/dev_requirements.txt @@ -4,4 +4,5 @@ ../azure-mgmt-communication aiohttp>=3.0 -e ../../../tools/azure-devtools -parameterized>=0.7.3 \ No newline at end of file +parameterized>=0.7.3 +python-dateutil>=2.8.0 \ No newline at end of file diff --git a/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py b/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py index 99df2c05c905c..f5cdf7425cae7 100644 --- a/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py +++ b/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py @@ -40,8 +40,6 @@ from requests import Request, Response -from msrest import Deserializer - from azure.core.polling import async_poller, AsyncLROPoller from azure.core.exceptions import DecodeError, HttpResponseError from azure.core import AsyncPipelineClient diff --git a/sdk/core/azure-core/tests/async_tests/test_polling_async.py b/sdk/core/azure-core/tests/async_tests/test_polling_async.py index ccee3335e48d0..469f30ba42f3f 100644 --- a/sdk/core/azure-core/tests/async_tests/test_polling_async.py +++ b/sdk/core/azure-core/tests/async_tests/test_polling_async.py @@ -35,7 +35,7 @@ from azure.core import AsyncPipelineClient from azure.core.polling import * from azure.core.exceptions import ServiceResponseError -from msrest.serialization import Model +# from msrest.serialization import Model @pytest.fixture @@ -145,9 +145,9 @@ def deserialization_callback(response): assert raw_poller.polling_method() is method done_cb.assert_called_once_with(poller) - # Test with a basic Model - poller = AsyncLROPoller(client, initial_response, Model, method) - assert poller._polling_method._deserialization_callback == Model.deserialize + # # Test with a basic Model + # poller = AsyncLROPoller(client, initial_response, Model, method) + # assert poller._polling_method._deserialization_callback == Model.deserialize # Test poller that method do a run method = PollingTwoSteps(sleep=1) diff --git a/sdk/core/azure-core/tests/test_polling.py b/sdk/core/azure-core/tests/test_polling.py index b01027f44a672..79549daadd1d9 100644 --- a/sdk/core/azure-core/tests/test_polling.py +++ b/sdk/core/azure-core/tests/test_polling.py @@ -37,7 +37,7 @@ from azure.core.polling.base_polling import ( LROBasePolling, LocationPolling ) -from msrest.serialization import Model +# from msrest.serialization import Model @pytest.fixture @@ -181,9 +181,9 @@ def deserialization_callback(response): assert poller.polling_method() is method done_cb.assert_called_once_with(method) - # Test with a basic Model - poller = LROPoller(client, initial_response, Model, method) - assert poller._polling_method._deserialization_callback == Model.deserialize + # # Test with a basic Model + # poller = LROPoller(client, initial_response, Model, method) + # assert poller._polling_method._deserialization_callback == Model.deserialize # Test poller that method do a run method = PollingTwoSteps(sleep=1) diff --git a/sdk/core/azure-mgmt-core/tests/asynctests/test_async_arm_polling.py b/sdk/core/azure-mgmt-core/tests/asynctests/test_async_arm_polling.py index a90f42103e4a5..2671d3a6c8b74 100644 --- a/sdk/core/azure-mgmt-core/tests/asynctests/test_async_arm_polling.py +++ b/sdk/core/azure-mgmt-core/tests/asynctests/test_async_arm_polling.py @@ -38,8 +38,6 @@ from requests import Request, Response -from msrest import Deserializer - from azure.core.polling import async_poller from azure.core.exceptions import DecodeError, HttpResponseError from azure.core import AsyncPipelineClient diff --git a/sdk/core/azure-mgmt-core/tests/test_arm_polling.py b/sdk/core/azure-mgmt-core/tests/test_arm_polling.py index 9e3783860ab6d..dc9dbc73c8d64 100644 --- a/sdk/core/azure-mgmt-core/tests/test_arm_polling.py +++ b/sdk/core/azure-mgmt-core/tests/test_arm_polling.py @@ -38,8 +38,6 @@ from requests import Request, Response -from msrest import Deserializer - from azure.core.polling import LROPoller from azure.core.exceptions import DecodeError, HttpResponseError from azure.core import PipelineClient diff --git a/sdk/identity/azure-identity/dev_requirements.txt b/sdk/identity/azure-identity/dev_requirements.txt index 56bb4a6b7a592..4011e5e83bffb 100644 --- a/sdk/identity/azure-identity/dev_requirements.txt +++ b/sdk/identity/azure-identity/dev_requirements.txt @@ -2,4 +2,5 @@ aiohttp>=3.0 typing_extensions>=3.7.2 -e ../../../tools/azure-sdk-tools --e ../../../tools/azure-devtools \ No newline at end of file +-e ../../../tools/azure-devtools +azure-mgmt-resource<=21.1.0 \ No newline at end of file diff --git a/sdk/keyvault/azure-keyvault-administration/dev_requirements.txt b/sdk/keyvault/azure-keyvault-administration/dev_requirements.txt index 76e717ccf8d9f..c572163678c1b 100644 --- a/sdk/keyvault/azure-keyvault-administration/dev_requirements.txt +++ b/sdk/keyvault/azure-keyvault-administration/dev_requirements.txt @@ -7,3 +7,4 @@ aiohttp>=3.0 azure-storage-blob==12.6.0 parameterized>=0.7.3 +python-dateutil>=2.8.0 \ No newline at end of file diff --git a/sdk/keyvault/azure-keyvault-certificates/dev_requirements.txt b/sdk/keyvault/azure-keyvault-certificates/dev_requirements.txt index ff343af4c8147..9da5daf6a5484 100644 --- a/sdk/keyvault/azure-keyvault-certificates/dev_requirements.txt +++ b/sdk/keyvault/azure-keyvault-certificates/dev_requirements.txt @@ -5,3 +5,4 @@ ../../nspkg/azure-keyvault-nspkg aiohttp>=3.0 parameterized>=0.7.3 +python-dateutil>=2.8.0 \ No newline at end of file diff --git a/sdk/keyvault/azure-keyvault-keys/dev_requirements.txt b/sdk/keyvault/azure-keyvault-keys/dev_requirements.txt index 8c60611c0b842..4e210aa7c2fb7 100644 --- a/sdk/keyvault/azure-keyvault-keys/dev_requirements.txt +++ b/sdk/keyvault/azure-keyvault-keys/dev_requirements.txt @@ -6,3 +6,4 @@ aiohttp>=3.0 azure-identity parameterized>=0.7.3 +python-dateutil>=2.8.0 \ No newline at end of file diff --git a/sdk/keyvault/azure-keyvault-secrets/dev_requirements.txt b/sdk/keyvault/azure-keyvault-secrets/dev_requirements.txt index 615d9dba4d5f1..49636d82810a4 100644 --- a/sdk/keyvault/azure-keyvault-secrets/dev_requirements.txt +++ b/sdk/keyvault/azure-keyvault-secrets/dev_requirements.txt @@ -5,3 +5,4 @@ ../../nspkg/azure-keyvault-nspkg aiohttp>=3.0 parameterized>=0.7.3 +python-dateutil>=2.8.0 \ No newline at end of file diff --git a/sdk/metricsadvisor/azure-ai-metricsadvisor/dev_requirements.txt b/sdk/metricsadvisor/azure-ai-metricsadvisor/dev_requirements.txt index a46c1ec382287..8847fdeb8f2db 100644 --- a/sdk/metricsadvisor/azure-ai-metricsadvisor/dev_requirements.txt +++ b/sdk/metricsadvisor/azure-ai-metricsadvisor/dev_requirements.txt @@ -2,4 +2,5 @@ -e ../../../tools/azure-devtools -e ../../identity/azure-identity ../../core/azure-core -aiohttp>=3.0 \ No newline at end of file +aiohttp>=3.0 +python-dateutil>=2.8.0 \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob/dev_requirements.txt b/sdk/storage/azure-storage-blob/dev_requirements.txt index a82ddde4e5ff1..43fb18a3329c8 100644 --- a/sdk/storage/azure-storage-blob/dev_requirements.txt +++ b/sdk/storage/azure-storage-blob/dev_requirements.txt @@ -4,4 +4,4 @@ ../../core/azure-core azure-identity -e ../../storage/azure-mgmt-storage -aiohttp>=3.0 +aiohttp>=3.0 \ No newline at end of file diff --git a/sdk/tables/azure-data-tables/dev_requirements.txt b/sdk/tables/azure-data-tables/dev_requirements.txt index b49a9a47b9882..8681731e014a3 100644 --- a/sdk/tables/azure-data-tables/dev_requirements.txt +++ b/sdk/tables/azure-data-tables/dev_requirements.txt @@ -6,3 +6,4 @@ ../../nspkg/azure-data-nspkg aiohttp>=3.0 azure-identity +python-dateutil>=2.8.0 \ No newline at end of file