include hooks in airflow setup.py

datahub-project · Nov 11, 2022 · 6e91c86 · 6e91c86
1 parent aa73660
commit 6e91c86
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -42,7 +42,7 @@ jobs:
           - python-version: "3.7"
             extraPythonRequirement: "sqlalchemy==1.3.24 apache-airflow~=2.2.0"
           - python-version: "3.10"
-            extraPythonRequirement: "sqlalchemy~=1.4.0 apache-airflow~=2.4.0"
+            extraPythonRequirement: "sqlalchemy~=1.4.0 apache-airflow>=2.4.0"
       fail-fast: false
     steps:
       - uses: actions/checkout@v3

diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
@@ -55,6 +55,10 @@ def get_long_description():
     "click-spinner",
 }
 
+rest_common = {
+    "requests",
+}
+
 kafka_common = {
     # The confluent_kafka package provides a number of pre-built wheels for
     # various platforms and architectures. However, it does not provide wheels
@@ -220,10 +224,12 @@ def get_long_description():
 plugins: Dict[str, Set[str]] = {
     # Sink plugins.
     "datahub-kafka": kafka_common,
-    "datahub-rest": {"requests"},
+    "datahub-rest": rest_common,
     # Integrations.
     "airflow": {
         "apache-airflow >= 2.0.2",
+        *rest_common,
+        *kafka_common,
     },
     "circuit-breaker": {
         "gql>=3.3.0",