From 9bd468d1f9b5dbd76d6eafa8fa40cb0c9f15d13f Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Fri, 31 Jan 2025 18:03:12 +0000 Subject: [PATCH] Set minimum dependencies for apache-beam on Py 3.12+3.13 (#46321) Older versions of apache-beam package depend upon gprcio-tools, but the version requirement they specify doesn't work on Py 3.12, but the metadata doesn't say that, so it tries to compile it and fails. This updates the min version to the min supported version with wheels for 3.12 --- generated/provider_dependencies.json | 3 ++- providers/apache/beam/README.rst | 9 +++++---- providers/apache/beam/pyproject.toml | 4 +++- .../airflow/providers/apache/beam/get_provider_info.py | 8 +++++++- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 434093bbce1db2..5b102eeaf1bd1e 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -75,7 +75,8 @@ "apache.beam": { "deps": [ "apache-airflow>=2.9.0", - "apache-beam>=2.53.0", + "apache-beam>=2.53.0; python_version < \"3.12\"", + "apache-beam>=2.57.0; python_version >= \"3.12\"", "numpy>=1.26.0", "pyarrow>=14.0.1" ], diff --git a/providers/apache/beam/README.rst b/providers/apache/beam/README.rst index 7f61ac4e600bd4..d55425f7cddae1 100644 --- a/providers/apache/beam/README.rst +++ b/providers/apache/beam/README.rst @@ -51,14 +51,15 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12 Requirements ------------ -================== ================== +================== ====================================== PIP package Version required -================== ================== +================== ====================================== ``apache-airflow`` ``>=2.9.0`` -``apache-beam`` ``>=2.53.0`` +``apache-beam`` ``>=2.53.0; python_version < "3.12"`` +``apache-beam`` ``>=2.57.0; python_version >= "3.12"`` ``pyarrow`` ``>=14.0.1`` ``numpy`` ``>=1.26.0`` -================== ================== +================== ====================================== Cross provider package dependencies ----------------------------------- diff --git a/providers/apache/beam/pyproject.toml b/providers/apache/beam/pyproject.toml index 9425bdfa5d3a18..9e8355d89a74f3 100644 --- a/providers/apache/beam/pyproject.toml +++ b/providers/apache/beam/pyproject.toml @@ -57,9 +57,11 @@ requires-python = "~=3.9" dependencies = [ "apache-airflow>=2.9.0", # Apache Beam > 2.53.0 and pyarrow > 14.0.1 fix https://nvd.nist.gov/vuln/detail/CVE-2023-47248. - "apache-beam>=2.53.0", + 'apache-beam>=2.53.0; python_version < "3.12"', + 'apache-beam>=2.57.0; python_version >= "3.12"', "pyarrow>=14.0.1", "numpy>=1.26.0", + ] # The optional dependencies should be modified in place in the generated file diff --git a/providers/apache/beam/src/airflow/providers/apache/beam/get_provider_info.py b/providers/apache/beam/src/airflow/providers/apache/beam/get_provider_info.py index 479cfbeb41ba07..4f9e2055564ebc 100644 --- a/providers/apache/beam/src/airflow/providers/apache/beam/get_provider_info.py +++ b/providers/apache/beam/src/airflow/providers/apache/beam/get_provider_info.py @@ -93,7 +93,13 @@ def get_provider_info(): "python-modules": ["airflow.providers.apache.beam.triggers.beam"], } ], - "dependencies": ["apache-airflow>=2.9.0", "apache-beam>=2.53.0", "pyarrow>=14.0.1", "numpy>=1.26.0"], + "dependencies": [ + "apache-airflow>=2.9.0", + 'apache-beam>=2.53.0; python_version < "3.12"', + 'apache-beam>=2.57.0; python_version >= "3.12"', + "pyarrow>=14.0.1", + "numpy>=1.26.0", + ], "optional-dependencies": { "google": ["apache-beam[gcp]"], "common.compat": ["apache-airflow-providers-common-compat"],