Pre commit script to validate template fields (#42284)

.pre-commit-config.yaml

@@ -1343,6 +1343,13 @@ repos:
         files: ^airflow/providers/.*/provider\.yaml$
         additional_dependencies: ['rich>=12.4.4']
         require_serial: true
+      - id: check-template-fields-valid
+        name: Check templated fields mapped in operators/sensors
+        language: python
+        entry: ./scripts/ci/pre_commit/check_template_fields.py
+        files: ^airflow/.*/sensors/.*\.py$|^airflow/.*/operators/.*\.py$
+        additional_dependencies: [ 'rich>=12.4.4' ]
+        require_serial: true
       - id: update-migration-references
         name: Update migration ref doc
         language: python

contributing-docs/08_static_code_checks.rst

@@ -236,6 +236,8 @@ require Breeze Docker image to be built locally.
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | check-template-context-variable-in-sync                   | Sync template context variable refs                    |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
+| check-template-fields-valid                               | Check templated fields mapped in operators/sensors     | *       |
++-----------------------------------------------------------+--------------------------------------------------------+---------+
 | check-tests-in-the-right-folders                          | Check if tests are in the right folders                |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | check-tests-unittest-testcase                             | Unit tests do not inherit from unittest.TestCase       |         |

dev/breeze/doc/images/output_static-checks.txt

@@ -1 +1 @@
-5c6ba60b1865538bce04fc940cd240c6
+e33cdf5f43d8c63290e44e92dc19d2c4

dev/breeze/src/airflow_breeze/pre_commit_ids.py

@@ -83,6 +83,7 @@
     "check-system-tests-tocs",
     "check-taskinstance-tis-attrs",
     "check-template-context-variable-in-sync",
+    "check-template-fields-valid",
     "check-tests-in-the-right-folders",
     "check-tests-unittest-testcase",
     "check-urlparse-usage-in-code",

scripts/ci/pre_commit/check_provider_yaml_files.py

@@ -17,12 +17,15 @@
 # under the License.
 from __future__ import annotations
 
-import os
 import sys
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.resolve()))
-from common_precommit_utils import console, initialize_breeze_precommit, run_command_via_breeze_shell
+from common_precommit_utils import (
+    initialize_breeze_precommit,
+    run_command_via_breeze_shell,
+    validate_cmd_result,
+)
 
 initialize_breeze_precommit(__name__, __file__)
 
@@ -33,10 +36,4 @@
     warn_image_upgrade_needed=True,
     extra_env={"PYTHONWARNINGS": "default"},
 )
-if cmd_result.returncode != 0 and os.environ.get("CI") != "true":
-    console.print(
-        "\n[yellow]If you see strange stacktraces above, especially about missing imports "
-        "run this command:[/]\n"
-    )
-    console.print("[magenta]breeze ci-image build --python 3.8 --upgrade-to-newer-dependencies[/]\n")
-sys.exit(cmd_result.returncode)
+validate_cmd_result(cmd_result, include_ci_env_check=True)

scripts/ci/pre_commit/check_template_fields.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.resolve()))
+from common_precommit_utils import (
+    initialize_breeze_precommit,
+    run_command_via_breeze_shell,
+    validate_cmd_result,
+)
+
+initialize_breeze_precommit(__name__, __file__)
+py_files_to_test = sys.argv[1:]
+
+cmd_result = run_command_via_breeze_shell(
+    ["python3", "/opt/airflow/scripts/in_container/run_template_fields_check.py", *py_files_to_test],
+    backend="sqlite",
+    warn_image_upgrade_needed=True,
+    extra_env={"PYTHONWARNINGS": "default"},
+)
+
+validate_cmd_result(cmd_result, include_ci_env_check=True)

scripts/ci/pre_commit/common_precommit_utils.py

@@ -211,3 +211,20 @@ def check_list_sorted(the_list: list[str], message: str, errors: list[str]) -> b
     console.print()
     errors.append(f"ERROR in {message}. The elements are not sorted/unique.")
     return False
+
+
+def validate_cmd_result(cmd_result, include_ci_env_check=False):
+    if include_ci_env_check:
+        if cmd_result.returncode != 0 and os.environ.get("CI") != "true":
+            console.print(
+                "\n[yellow]If you see strange stacktraces above, especially about missing imports "
+                "run this command:[/]\n"
+            )
+            console.print("[magenta]breeze ci-image build --python 3.8 --upgrade-to-newer-dependencies[/]\n")
+
+    elif cmd_result.returncode != 0:
+        console.print(
+            "[warning]\nIf you see strange stacktraces above, "
+            "run `breeze ci-image build --python 3.8` and try again."
+        )
+    sys.exit(cmd_result.returncode)

scripts/ci/pre_commit/migration_reference.py

@@ -21,17 +21,17 @@
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.resolve()))
-from common_precommit_utils import console, initialize_breeze_precommit, run_command_via_breeze_shell
+from common_precommit_utils import (
+    initialize_breeze_precommit,
+    run_command_via_breeze_shell,
+    validate_cmd_result,
+)
 
 initialize_breeze_precommit(__name__, __file__)
 
 cmd_result = run_command_via_breeze_shell(
     ["python3", "/opt/airflow/scripts/in_container/run_migration_reference.py"],
     backend="sqlite",
 )
-if cmd_result.returncode != 0:
-    console.print(
-        "[warning]\nIf you see strange stacktraces above, "
-        "run `breeze ci-image build --python 3.8` and try again."
-    )
-sys.exit(cmd_result.returncode)
+
+validate_cmd_result(cmd_result)

scripts/ci/pre_commit/update_er_diagram.py

@@ -21,7 +21,11 @@
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.resolve()))
-from common_precommit_utils import console, initialize_breeze_precommit, run_command_via_breeze_shell
+from common_precommit_utils import (
+    initialize_breeze_precommit,
+    run_command_via_breeze_shell,
+    validate_cmd_result,
+)
 
 initialize_breeze_precommit(__name__, __file__)
 
@@ -36,9 +40,4 @@
     },
 )
 
-if cmd_result.returncode != 0:
-    console.print(
-        "[warning]\nIf you see strange stacktraces above, "
-        "run `breeze ci-image build --python 3.8` and try again."
-    )
-    sys.exit(cmd_result.returncode)
+validate_cmd_result(cmd_result)

scripts/ci/pre_commit/update_fastapi_api_spec.py

@@ -21,7 +21,11 @@
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.resolve()))
-from common_precommit_utils import console, initialize_breeze_precommit, run_command_via_breeze_shell
+from common_precommit_utils import (
+    initialize_breeze_precommit,
+    run_command_via_breeze_shell,
+    validate_cmd_result,
+)
 
 initialize_breeze_precommit(__name__, __file__)
 
@@ -31,9 +35,4 @@
     skip_environment_initialization=False,
 )
 
-if cmd_result.returncode != 0:
-    console.print(
-        "[warning]\nIf you see strange stacktraces above, "
-        "run `breeze ci-image build --python 3.8` and try again."
-    )
-sys.exit(cmd_result.returncode)
+validate_cmd_result(cmd_result)