Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[airflow]: extend names moved from core to provider (AIR303) #15159

Merged
merged 1 commit into from
Dec 27, 2024
Merged

[airflow]: extend names moved from core to provider (AIR303) #15159

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 27 additions & 3 deletions crates/ruff_linter/resources/test/fixtures/airflow/AIR303.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,41 @@
from airflow.auth.managers.fab.api.auth.backend import (
kerberos_auth as backend_kerberos_auth,
)
from airflow.config_templates.default_celery import DEFAULT_CELERY_CONFIG
from airflow.auth.managers.fab.fab_auth_manager import FabAuthManager
from airflow.auth.managers.fab.security_manager import override as fab_override
from airflow.config_templates.default_celery import DEFAULT_CELERY_CONFIG
from airflow.executors.celery_executor import app
from airflow.executors.kubernetes_executor_types import (
ALL_NAMESPACES,
POD_EXECUTOR_DONE_KEY,
)
from airflow.hooks.dbapi import ConnectorProtocol, DbApiHook
from airflow.hooks.hive_hooks import HIVE_QUEUE_PRIORITIES
from airflow.macros.hive import closest_ds_partition, max_partition
from airflow.www.security import FabAirflowSecurityManagerOverride

# apache-airflow-providers-fab
basic_auth, kerberos_auth
auth_current_user
backend_kerberos_auth
fab_override

FabAuthManager
FabAirflowSecurityManagerOverride
FabAuthManager()
FabAirflowSecurityManagerOverride()

# apache-airflow-providers-celery
DEFAULT_CELERY_CONFIG
app

# apache-airflow-providers-common-sql
ConnectorProtocol()
DbApiHook()

# apache-airflow-providers-cncf-kubernetes
ALL_NAMESPACES
POD_EXECUTOR_DONE_KEY

# apache-airflow-providers-apache-hive
HIVE_QUEUE_PRIORITIES
closest_ds_partition()
max_partition()
76 changes: 75 additions & 1 deletion crates/ruff_linter/src/rules/airflow/rules/moved_to_provider_in_3.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,81 @@ fn moved_to_provider(checker: &mut Checker, expr: &Expr, ranged: impl Ranged) {
version: "3.3.0"
},
)),

["airflow", "executors", "celery_executor", "app"] => Some((
qualname.to_string(),
Replacement::ImportPathMoved{
original_path: "airflow.executors.celery_executor.app",
new_path: "airflow.providers.celery.executors.celery_executor_utils.app",
provider: "celery",
version: "3.3.0"
},
)),
// apache-airflow-providers-common-sql
["airflow", "hooks", "dbapi", "ConnectorProtocol"] => Some((
qualname.to_string(),
Replacement::ImportPathMoved{
original_path: "airflow.hooks.dbapi.ConnectorProtocol",
new_path: "airflow.providers.common.sql.hooks.sql.ConnectorProtocol",
provider: "Common SQL",
version: "1.0.0"
},
)),
["airflow", "hooks", "dbapi", "DbApiHook"] => Some((
qualname.to_string(),
Replacement::ImportPathMoved{
original_path: "airflow.hooks.dbapi.DbApiHook",
new_path: "airflow.providers.common.sql.hooks.sql.DbApiHook",
provider: "Common SQL",
version: "1.0.0"
},
)),
// apache-airflow-providers-cncf-kubernetes
["airflow", "executors", "kubernetes_executor_types", "ALL_NAMESPACES"] => Some((
qualname.to_string(),
Replacement::ImportPathMoved{
original_path: "airflow.executors.kubernetes_executor_types.ALL_NAMESPACES",
new_path: "airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types.ALL_NAMESPACES",
provider: "Kubernetes",
version: "7.4.0"
},
)),
["airflow", "executors", "kubernetes_executor_types", "POD_EXECUTOR_DONE_KEY"] => Some((
qualname.to_string(),
Replacement::ImportPathMoved{
original_path: "airflow.executors.kubernetes_executor_types.POD_EXECUTOR_DONE_KEY",
new_path: "airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types.POD_EXECUTOR_DONE_KEY",
provider: "Kubernetes",
version: "7.4.0"
},
)),
// apache-airflow-providers-apache-hive
["airflow", "hooks", "hive_hooks", "HIVE_QUEUE_PRIORITIES"] => Some((
qualname.to_string(),
Replacement::ImportPathMoved{
original_path: "airflow.hooks.hive_hooks.HIVE_QUEUE_PRIORITIES",
new_path: "airflow.providers.apache.hive.hooks.hive.HIVE_QUEUE_PRIORITIES",
provider: "Apache Hive",
version: "1.0.0"
},
)),
["airflow", "macros", "hive", "closest_ds_partition"] => Some((
qualname.to_string(),
Replacement::ImportPathMoved{
original_path: "airflow.macros.hive.closest_ds_partition",
new_path: "airflow.providers.apache.hive.macros.hive.closest_ds_partition",
provider: "Apache Hive",
version: "5.1.0"
},
)),
["airflow", "macros", "hive", "max_partition"] => Some((
qualname.to_string(),
Replacement::ImportPathMoved{
original_path: "airflow.macros.hive.max_partition",
new_path: "airflow.providers.apache.hive.macros.hive.max_partition",
provider: "Apache Hive",
version: "5.1.0"
},
)),
_ => None,
});
if let Some((deprecated, replacement)) = result {
Expand Down
163 changes: 121 additions & 42 deletions ...ter/src/rules/airflow/snapshots/ruff_linter__rules__airflow__tests__AIR303_AIR303.py.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,84 +2,163 @@
source: crates/ruff_linter/src/rules/airflow/mod.rs
snapshot_kind: text
---
AIR303.py:11:1: AIR303 Import path `airflow.api.auth.backend.basic_auth` is moved into `fab` provider in Airflow 3.0;
AIR303.py:20:1: AIR303 Import path `airflow.api.auth.backend.basic_auth` is moved into `fab` provider in Airflow 3.0;
|
9 | from airflow.www.security import FabAirflowSecurityManagerOverride
10 |
11 | basic_auth, kerberos_auth
19 | # apache-airflow-providers-fab
20 | basic_auth, kerberos_auth
| ^^^^^^^^^^ AIR303
12 | auth_current_user
13 | backend_kerberos_auth
21 | auth_current_user
22 | backend_kerberos_auth
|
= help: Install `apache-airflow-provider-fab>=1.0.0` and import from `airflow.providers.fab.auth_manager.api.auth.backend.basic_auth` instead.

AIR303.py:11:13: AIR303 Import path `airflow.api.auth.backend.kerberos_auth` is moved into `fab` provider in Airflow 3.0;
AIR303.py:20:13: AIR303 Import path `airflow.api.auth.backend.kerberos_auth` is moved into `fab` provider in Airflow 3.0;
|
9 | from airflow.www.security import FabAirflowSecurityManagerOverride
10 |
11 | basic_auth, kerberos_auth
19 | # apache-airflow-providers-fab
20 | basic_auth, kerberos_auth
| ^^^^^^^^^^^^^ AIR303
12 | auth_current_user
13 | backend_kerberos_auth
21 | auth_current_user
22 | backend_kerberos_auth
|
= help: Install `apache-airflow-provider-fab>=1.0.0` and import from `airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth` instead.

AIR303.py:12:1: AIR303 Import path `airflow.api.auth.backend.basic_auth` is moved into `fab` provider in Airflow 3.0;
AIR303.py:21:1: AIR303 Import path `airflow.api.auth.backend.basic_auth` is moved into `fab` provider in Airflow 3.0;
|
11 | basic_auth, kerberos_auth
12 | auth_current_user
19 | # apache-airflow-providers-fab
20 | basic_auth, kerberos_auth
21 | auth_current_user
| ^^^^^^^^^^^^^^^^^ AIR303
13 | backend_kerberos_auth
14 | fab_override
22 | backend_kerberos_auth
23 | fab_override
|
= help: Install `apache-airflow-provider-fab>=1.0.0` and import from `airflow.providers.fab.auth_manager.api.auth.backend.basic_auth` instead.

AIR303.py:13:1: AIR303 Import path `airflow.auth_manager.api.auth.backend.kerberos_auth` is moved into `fab` provider in Airflow 3.0;
AIR303.py:22:1: AIR303 Import path `airflow.auth_manager.api.auth.backend.kerberos_auth` is moved into `fab` provider in Airflow 3.0;
|
11 | basic_auth, kerberos_auth
12 | auth_current_user
13 | backend_kerberos_auth
20 | basic_auth, kerberos_auth
21 | auth_current_user
22 | backend_kerberos_auth
| ^^^^^^^^^^^^^^^^^^^^^ AIR303
14 | fab_override
23 | fab_override
|
= help: Install `apache-airflow-provider-fab>=1.0.0` and import from `airflow.providers.fab.auth_manager.api.auth.backend.kerberos_auth` instead.

AIR303.py:14:1: AIR303 Import path `airflow.auth.managers.fab.security_managr.override` is moved into `fab` provider in Airflow 3.0;
AIR303.py:23:1: AIR303 Import path `airflow.auth.managers.fab.security_managr.override` is moved into `fab` provider in Airflow 3.0;
|
12 | auth_current_user
13 | backend_kerberos_auth
14 | fab_override
21 | auth_current_user
22 | backend_kerberos_auth
23 | fab_override
| ^^^^^^^^^^^^ AIR303
15 |
16 | FabAuthManager
24 |
25 | FabAuthManager()
|
= help: Install `apache-airflow-provider-fab>=1.0.0` and import from `airflow.providers.fab.auth_manager.security_manager.override` instead.

AIR303.py:16:1: AIR303 `airflow.auth.managers.fab.fab_auth_manager.FabAuthManager` is moved into `fab` provider in Airflow 3.0;
AIR303.py:25:1: AIR303 `airflow.auth.managers.fab.fab_auth_manager.FabAuthManager` is moved into `fab` provider in Airflow 3.0;
|
14 | fab_override
15 |
16 | FabAuthManager
23 | fab_override
24 |
25 | FabAuthManager()
| ^^^^^^^^^^^^^^ AIR303
17 | FabAirflowSecurityManagerOverride
26 | FabAirflowSecurityManagerOverride()
|
= help: Install `apache-airflow-provider-fab>=1.0.0` and use `airflow.providers.fab.auth_manager.security_manager.FabAuthManager` instead.

AIR303.py:17:1: AIR303 `airflow.www.security.FabAirflowSecurityManagerOverride` is moved into `fab` provider in Airflow 3.0;
AIR303.py:26:1: AIR303 `airflow.www.security.FabAirflowSecurityManagerOverride` is moved into `fab` provider in Airflow 3.0;
|
16 | FabAuthManager
17 | FabAirflowSecurityManagerOverride
25 | FabAuthManager()
26 | FabAirflowSecurityManagerOverride()
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AIR303
18 |
19 | DEFAULT_CELERY_CONFIG
27 |
28 | # apache-airflow-providers-celery
|
= help: Install `apache-airflow-provider-fab>=1.0.0` and use `airflow.providers.fab.auth_manager.security_manager.override.FabAirflowSecurityManagerOverride` instead.

AIR303.py:19:1: AIR303 Import path `airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG` is moved into `celery` provider in Airflow 3.0;
AIR303.py:29:1: AIR303 Import path `airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG` is moved into `celery` provider in Airflow 3.0;
|
17 | FabAirflowSecurityManagerOverride
18 |
19 | DEFAULT_CELERY_CONFIG
28 | # apache-airflow-providers-celery
29 | DEFAULT_CELERY_CONFIG
| ^^^^^^^^^^^^^^^^^^^^^ AIR303
30 | app
|
= help: Install `apache-airflow-provider-celery>=3.3.0` and import from `airflow.providers.celery.executors.default_celery.DEFAULT_CELERY_CONFIG` instead.

AIR303.py:30:1: AIR303 Import path `airflow.executors.celery_executor.app` is moved into `celery` provider in Airflow 3.0;
|
28 | # apache-airflow-providers-celery
29 | DEFAULT_CELERY_CONFIG
30 | app
| ^^^ AIR303
31 |
32 | # apache-airflow-providers-common-sql
|
= help: Install `apache-airflow-provider-celery>=3.3.0` and import from `airflow.providers.celery.executors.celery_executor_utils.app` instead.

AIR303.py:33:1: AIR303 Import path `airflow.hooks.dbapi.ConnectorProtocol` is moved into `Common SQL` provider in Airflow 3.0;
|
32 | # apache-airflow-providers-common-sql
33 | ConnectorProtocol()
| ^^^^^^^^^^^^^^^^^ AIR303
34 | DbApiHook()
|
= help: Install `apache-airflow-provider-Common SQL>=1.0.0` and import from `airflow.providers.common.sql.hooks.sql.ConnectorProtocol` instead.

AIR303.py:34:1: AIR303 Import path `airflow.hooks.dbapi.DbApiHook` is moved into `Common SQL` provider in Airflow 3.0;
|
32 | # apache-airflow-providers-common-sql
33 | ConnectorProtocol()
34 | DbApiHook()
| ^^^^^^^^^ AIR303
35 |
36 | # apache-airflow-providers-cncf-kubernetes
|
= help: Install `apache-airflow-provider-Common SQL>=1.0.0` and import from `airflow.providers.common.sql.hooks.sql.DbApiHook` instead.

AIR303.py:37:1: AIR303 Import path `airflow.executors.kubernetes_executor_types.ALL_NAMESPACES` is moved into `Kubernetes` provider in Airflow 3.0;
|
36 | # apache-airflow-providers-cncf-kubernetes
37 | ALL_NAMESPACES
| ^^^^^^^^^^^^^^ AIR303
38 | POD_EXECUTOR_DONE_KEY
|
= help: Install `apache-airflow-provider-Kubernetes>=7.4.0` and import from `airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types.ALL_NAMESPACES` instead.

AIR303.py:38:1: AIR303 Import path `airflow.executors.kubernetes_executor_types.POD_EXECUTOR_DONE_KEY` is moved into `Kubernetes` provider in Airflow 3.0;
|
36 | # apache-airflow-providers-cncf-kubernetes
37 | ALL_NAMESPACES
38 | POD_EXECUTOR_DONE_KEY
| ^^^^^^^^^^^^^^^^^^^^^ AIR303
39 |
40 | # apache-airflow-providers-apache-hive
|
= help: Install `apache-airflow-provider-Kubernetes>=7.4.0` and import from `airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types.POD_EXECUTOR_DONE_KEY` instead.

AIR303.py:41:1: AIR303 Import path `airflow.hooks.hive_hooks.HIVE_QUEUE_PRIORITIES` is moved into `Apache Hive` provider in Airflow 3.0;
|
40 | # apache-airflow-providers-apache-hive
41 | HIVE_QUEUE_PRIORITIES
| ^^^^^^^^^^^^^^^^^^^^^ AIR303
42 | closest_ds_partition()
43 | max_partition()
|
= help: Install `apache-airflow-provider-Apache Hive>=1.0.0` and import from `airflow.providers.apache.hive.hooks.hive.HIVE_QUEUE_PRIORITIES` instead.

AIR303.py:42:1: AIR303 Import path `airflow.macros.hive.closest_ds_partition` is moved into `Apache Hive` provider in Airflow 3.0;
|
40 | # apache-airflow-providers-apache-hive
41 | HIVE_QUEUE_PRIORITIES
42 | closest_ds_partition()
| ^^^^^^^^^^^^^^^^^^^^ AIR303
43 | max_partition()
|
= help: Install `apache-airflow-provider-Apache Hive>=5.1.0` and import from `airflow.providers.apache.hive.macros.hive.closest_ds_partition` instead.

AIR303.py:43:1: AIR303 Import path `airflow.macros.hive.max_partition` is moved into `Apache Hive` provider in Airflow 3.0;
|
41 | HIVE_QUEUE_PRIORITIES
42 | closest_ds_partition()
43 | max_partition()
| ^^^^^^^^^^^^^ AIR303
|
= help: Install `apache-airflow-provider-Apache Hive>=5.1.0` and import from `airflow.providers.apache.hive.macros.hive.max_partition` instead.
Loading