From 8a06d5050bff6c7aaec81874cc107a5f24e47136 Mon Sep 17 00:00:00 2001 From: jason810496 Date: Fri, 3 Jan 2025 19:49:22 +0800 Subject: [PATCH 1/4] Move airflow.lineage.entities to compact provider --- .../src/airflow/providers/common/compat}/lineage/entities.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {airflow => providers/src/airflow/providers/common/compat}/lineage/entities.py (100%) diff --git a/airflow/lineage/entities.py b/providers/src/airflow/providers/common/compat/lineage/entities.py similarity index 100% rename from airflow/lineage/entities.py rename to providers/src/airflow/providers/common/compat/lineage/entities.py From ea3cfa2f253bc4f302ebb5a008ad17104758e2c5 Mon Sep 17 00:00:00 2001 From: jason810496 Date: Fri, 3 Jan 2025 19:54:53 +0800 Subject: [PATCH 2/4] Fix providers and corresponding test import path --- generated/provider_dependencies.json | 4 +++- .../src/airflow/providers/openlineage/extractors/manager.py | 4 ++-- .../src/airflow/providers/papermill/operators/papermill.py | 2 +- providers/tests/openlineage/extractors/test_manager.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 5060869ea3c6a..7ae6ab718abb1 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -1047,7 +1047,9 @@ ], "devel-deps": [], "plugins": [], - "cross-providers-deps": [], + "cross-providers-deps": [ + "common.compat" + ], "excluded-python-versions": [], "state": "ready" }, diff --git a/providers/src/airflow/providers/openlineage/extractors/manager.py b/providers/src/airflow/providers/openlineage/extractors/manager.py index 12d64050208be..f07014885eaff 100644 --- a/providers/src/airflow/providers/openlineage/extractors/manager.py +++ b/providers/src/airflow/providers/openlineage/extractors/manager.py @@ -36,8 +36,8 @@ if TYPE_CHECKING: from openlineage.client.event_v2 import Dataset - from airflow.lineage.entities import Table from airflow.models import Operator + from airflow.providers.common.compat.lineage.entities import Table def _iter_extractor_types() -> Iterator[type[BaseExtractor]]: @@ -293,7 +293,7 @@ def convert_to_ol_dataset_from_table(table: Table) -> Dataset: def convert_to_ol_dataset(obj) -> Dataset | None: from openlineage.client.event_v2 import Dataset - from airflow.lineage.entities import File, Table + from airflow.providers.common.compat.lineage.entities import File, Table if isinstance(obj, Dataset): return obj diff --git a/providers/src/airflow/providers/papermill/operators/papermill.py b/providers/src/airflow/providers/papermill/operators/papermill.py index b07c83d048f9b..7384e204ea1a2 100644 --- a/providers/src/airflow/providers/papermill/operators/papermill.py +++ b/providers/src/airflow/providers/papermill/operators/papermill.py @@ -24,8 +24,8 @@ import attr import papermill as pm -from airflow.lineage.entities import File from airflow.models import BaseOperator +from airflow.providers.common.compat.lineage.entities import File from airflow.providers.papermill.hooks.kernel import REMOTE_KERNEL_ENGINE, KernelHook if TYPE_CHECKING: diff --git a/providers/tests/openlineage/extractors/test_manager.py b/providers/tests/openlineage/extractors/test_manager.py index df64b7d1e75ef..b9595f80b91c8 100644 --- a/providers/tests/openlineage/extractors/test_manager.py +++ b/providers/tests/openlineage/extractors/test_manager.py @@ -30,9 +30,9 @@ ) from airflow.io.path import ObjectStoragePath -from airflow.lineage.entities import Column, File, Table, User from airflow.models.baseoperator import BaseOperator from airflow.models.taskinstance import TaskInstance +from airflow.providers.common.compat.lineage.entities import Column, File, Table, User from airflow.providers.openlineage.extractors import OperatorLineage from airflow.providers.openlineage.extractors.manager import ExtractorManager from airflow.providers.openlineage.utils.utils import Asset From 6fbdb06e79a6efd3b0fbaaf39f1c10d3e4dbc95f Mon Sep 17 00:00:00 2001 From: jason810496 Date: Fri, 3 Jan 2025 19:57:59 +0800 Subject: [PATCH 3/4] Fix core test import path --- tests/lineage/test_lineage.py | 2 +- tests/models/test_baseoperator.py | 2 +- tests/www/views/test_views_grid.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/lineage/test_lineage.py b/tests/lineage/test_lineage.py index d805d3f5ec48e..7597cd6069f28 100644 --- a/tests/lineage/test_lineage.py +++ b/tests/lineage/test_lineage.py @@ -24,10 +24,10 @@ from airflow.lineage import AUTO, apply_lineage, get_backend, prepare_lineage from airflow.lineage.backend import LineageBackend -from airflow.lineage.entities import File from airflow.models import TaskInstance as TI from airflow.operators.empty import EmptyOperator from airflow.sdk.definitions.context import Context +from airflow.providers.common.compat.lineage.entities import File from airflow.utils import timezone from airflow.utils.types import DagRunType diff --git a/tests/models/test_baseoperator.py b/tests/models/test_baseoperator.py index 13dad2bc8a4c3..ee45fb02933fb 100644 --- a/tests/models/test_baseoperator.py +++ b/tests/models/test_baseoperator.py @@ -26,7 +26,6 @@ from airflow.decorators import task as task_decorator from airflow.exceptions import AirflowException, TaskDeferralTimeout -from airflow.lineage.entities import File from airflow.models.baseoperator import ( BaseOperator, chain, @@ -37,6 +36,7 @@ from airflow.models.dagrun import DagRun from airflow.models.taskinstance import TaskInstance from airflow.models.trigger import TriggerFailureReason +from airflow.providers.common.compat.lineage.entities import File from airflow.providers.common.sql.operators import sql from airflow.utils.edgemodifier import Label from airflow.utils.task_group import TaskGroup diff --git a/tests/www/views/test_views_grid.py b/tests/www/views/test_views_grid.py index e2181aa702d25..475c9ee63ffa1 100644 --- a/tests/www/views/test_views_grid.py +++ b/tests/www/views/test_views_grid.py @@ -25,10 +25,10 @@ from dateutil.tz import UTC from airflow.decorators import task_group -from airflow.lineage.entities import File from airflow.models import DagBag from airflow.models.asset import AssetDagRunQueue, AssetEvent, AssetModel from airflow.operators.empty import EmptyOperator +from airflow.providers.common.compat.lineage.entities import File from airflow.sdk.definitions.asset import Asset from airflow.utils import timezone from airflow.utils.state import DagRunState, TaskInstanceState From 7158e1eb85eb5555b857d224d482710fd7697de0 Mon Sep 17 00:00:00 2001 From: jason810496 Date: Thu, 23 Jan 2025 21:12:09 +0800 Subject: [PATCH 4/4] Fix import path in related docs --- docs/apache-airflow-providers-openlineage/guides/developer.rst | 2 +- docs/apache-airflow/administration-and-deployment/lineage.rst | 2 +- tests/lineage/test_lineage.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/apache-airflow-providers-openlineage/guides/developer.rst b/docs/apache-airflow-providers-openlineage/guides/developer.rst index 4eca316217ef9..69dd8bfdf2c4c 100644 --- a/docs/apache-airflow-providers-openlineage/guides/developer.rst +++ b/docs/apache-airflow-providers-openlineage/guides/developer.rst @@ -387,8 +387,8 @@ An Operator inside the Airflow DAG can be annotated with inlets and outlets like import pendulum from airflow import DAG + from airflow.providers.common.compat.lineage.entities import Table, File, Column, User from airflow.providers.standard.operators.bash import BashOperator - from airflow.lineage.entities import Table, File, Column, User t1 = Table( diff --git a/docs/apache-airflow/administration-and-deployment/lineage.rst b/docs/apache-airflow/administration-and-deployment/lineage.rst index 3740e8b56f937..836b11fe720f6 100644 --- a/docs/apache-airflow/administration-and-deployment/lineage.rst +++ b/docs/apache-airflow/administration-and-deployment/lineage.rst @@ -34,8 +34,8 @@ works. import pendulum from airflow.lineage import AUTO - from airflow.lineage.entities import File from airflow.models import DAG + from airflow.providers.common.compat.lineage.entities import File from airflow.providers.standard.operators.bash import BashOperator from airflow.operators.empty import EmptyOperator diff --git a/tests/lineage/test_lineage.py b/tests/lineage/test_lineage.py index 7597cd6069f28..069d74f540acc 100644 --- a/tests/lineage/test_lineage.py +++ b/tests/lineage/test_lineage.py @@ -26,8 +26,8 @@ from airflow.lineage.backend import LineageBackend from airflow.models import TaskInstance as TI from airflow.operators.empty import EmptyOperator -from airflow.sdk.definitions.context import Context from airflow.providers.common.compat.lineage.entities import File +from airflow.sdk.definitions.context import Context from airflow.utils import timezone from airflow.utils.types import DagRunType