dev: update pre-commit config (datahub-project#12592)

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-01-17 16:43:31 UTC
+# Auto-generated by .github/scripts/generate_pre_commit.py at 2025-02-11 10:00:11 UTC
 # Do not edit this file directly. Run the script to regenerate.
 # Add additional hooks in .github/scripts/pre-commit-override.yaml
 repos:
@@ -53,13 +53,6 @@ repos:
         files: ^metadata-dao-impl/kafka-producer/.*\.java$
         pass_filenames: false
 
-      - id: metadata-events-mxe-avro-spotless
-        name: metadata-events/mxe-avro Spotless Apply
-        entry: ./gradlew :metadata-events:mxe-avro:spotlessApply
-        language: system
-        files: ^metadata-events/mxe-avro/.*\.java$
-        pass_filenames: false
-
       - id: metadata-events-mxe-registration-spotless
         name: metadata-events/mxe-registration Spotless Apply
         entry: ./gradlew :metadata-events:mxe-registration:spotlessApply
@@ -312,6 +305,20 @@ repos:
         files: ^metadata-service/graphql-servlet-impl/.*\.java$
         pass_filenames: false
 
+      - id: metadata-service-iceberg-catalog-spotless
+        name: metadata-service/iceberg-catalog Spotless Apply
+        entry: ./gradlew :metadata-service:iceberg-catalog:spotlessApply
+        language: system
+        files: ^metadata-service/iceberg-catalog/.*\.java$
+        pass_filenames: false
+
+      - id: metadata-service-iceberg-catalog-lint-fix
+        name: metadata-service/iceberg-catalog Lint Fix
+        entry: ./gradlew :metadata-service:iceberg-catalog:lintFix
+        language: system
+        files: ^metadata-service/iceberg-catalog/.*\.py$
+        pass_filenames: false
+
       - id: metadata-service-openapi-analytics-servlet-spotless
         name: metadata-service/openapi-analytics-servlet Spotless Apply
         entry: ./gradlew :metadata-service:openapi-analytics-servlet:spotlessApply
@@ -410,6 +417,13 @@ repos:
         files: ^metadata-service/servlet/.*\.java$
         pass_filenames: false
 
+      - id: metadata-service-war-spotless
+        name: metadata-service/war Spotless Apply
+        entry: ./gradlew :metadata-service:war:spotlessApply
+        language: system
+        files: ^metadata-service/war/.*\.java$
+        pass_filenames: false
+
       - id: metadata-utils-spotless
         name: metadata-utils Spotless Apply
         entry: ./gradlew :metadata-utils:spotlessApply

metadata-service/iceberg-catalog/build.gradle

@@ -82,3 +82,11 @@ task integrationTest(type: Exec, dependsOn: installDev) {
         "source ${venv_name}/bin/activate && set -x && " +
             "pytest"
 }
+
+
+task lintFix(type: Exec, dependsOn: installDev) {
+  commandLine 'bash', '-c',
+    "source ${venv_name}/bin/activate && set -x && " +
+    "ruff check --fix src/integrationTest/ && " +
+    "ruff format src/integrationTest/ "
+}

metadata-service/iceberg-catalog/pyproject.toml

@@ -25,7 +25,7 @@ target-version = ['py310']
 [tool.isort]
 profile = 'black'
 
-[tool.ruff]
+[tool.ruff.lint]
 ignore = [
 	'E501', # Ignore line length, since black handles that.
 	'D203', # Ignore 1 blank line required before class docstring.

metadata-service/iceberg-catalog/requirements.txt

@@ -6,7 +6,7 @@ pyspark==3.5.3
 black==24.3.0
 isort==5.12.0
 mypy==1.5.1
-ruff==0.0.287
+ruff==0.9.6
 # stub version are copied from metadata-ingestion/setup.py and that should be the source of truth
 types-requests>=2.28.11.6,<=2.31.0.3
 types-PyYAML

metadata-service/iceberg-catalog/src/integrationTest/integration_test.py

@@ -2,8 +2,8 @@
 import os
 from urllib.parse import urlparse
 import pytest
-from datahub.cli import cli_utils, env_utils, iceberg_cli
-from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
+from datahub.cli import cli_utils, iceberg_cli
+from datahub.ingestion.graph.client import get_default_graph
 
 
 def get_gms_url():
@@ -48,9 +48,7 @@ def give_all_permissions(username, policy_name):
         """
     variables = {"user": f"urn:li:corpuser:{username}", "policyName": policy_name}
 
-    response = client.execute_graphql(
-        query, variables=variables, format_exception=False
-    )
+    client.execute_graphql(query, variables=variables, format_exception=False)
 
 
 @pytest.fixture
@@ -163,7 +161,7 @@ def _test_basic_table_ops(spark_session):
     try:
         spark_session.sql("select * from test_table")
         assert False, "Table must not exist"
-    except:
+    except Exception:
         pass  # Exception is expected
 
     # TODO: Add dataset verification
@@ -181,7 +179,7 @@ def _test_basic_view_ops(spark_session):
     try:
         spark_session.sql("SELECT * FROM test_view")
         assert False, "test_view must not exist"
-    except:
+    except Exception:
         pass  # Exception is expected
 
     spark_session.sql("drop table test_table")
@@ -196,7 +194,7 @@ def _test_rename_ops(spark_session):
     try:
         spark_session.sql("SELECT * FROM test_table")
         assert False, "test_table must not exist"
-    except:
+    except Exception:
         pass  # Exception is expected
 
     spark_session.sql("insert into test_table_renamed values(2, 'bar' ) ")