[BUGFIX] Patch inconsistent ordering within GCP test asserts (#7130)

Co-authored-by: Will Shin <[email protected]>
great-expectations · Feb 14, 2023 · 6f47d02 · 6f47d02
1 parent 445d003
commit 6f47d02
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 20 deletions.
diff --git a/great_expectations/self_check/util.py b/great_expectations/self_check/util.py
@@ -40,13 +40,6 @@
     IDDict,
 )
 from great_expectations.core.batch import Batch, BatchDefinition, BatchRequest
-from great_expectations.core.expectation_diagnostics.expectation_test_data_cases import (
-    ExpectationTestCase,  # noqa: TCH001
-    ExpectationTestDataCases,  # noqa: TCH001
-)
-from great_expectations.core.expectation_diagnostics.supporting_types import (
-    ExpectationExecutionEngineDiagnostics,  # noqa: TCH001
-)
 from great_expectations.core.util import (
     get_or_create_spark_application,
     get_sql_dialect_floating_point_infinity_value,
@@ -79,6 +72,13 @@
 from great_expectations.validator.validator import Validator
 
 if TYPE_CHECKING:
+    from great_expectations.core.expectation_diagnostics.expectation_test_data_cases import (
+        ExpectationTestCase,
+        ExpectationTestDataCases,
+    )
+    from great_expectations.core.expectation_diagnostics.supporting_types import (
+        ExpectationExecutionEngineDiagnostics,
+    )
     from great_expectations.data_context import AbstractDataContext
 
 expectationValidationResultSchema = ExpectationValidationResultSchema()
@@ -3064,10 +3064,12 @@ def check_json_test_result(  # noqa: C901 - 52
                 assert result["result"]["observed_value"] in value
 
             elif key == "unexpected_index_list":
-                if pk_column and result["result"].get("unexpected_index_list"):
+                unexpected_list = result["result"].get("unexpected_index_list")
+                if pk_column and unexpected_list:
+                    # Note that consistent ordering of unexpected_list is not a guarantee by ID/PK
                     assert (
-                        result["result"].get("unexpected_index_list") == value
-                    ), f"{result['result'].get('unexpected_index_list')} != {value}"
+                        sorted(unexpected_list, key=lambda d: d["pk_index"]) == value
+                    ), f"{unexpected_list} != {value}"
 
             elif key == "unexpected_list":
                 try:

diff --git a/...ns/column_map_expectations/expect_column_values_to_be_between__evaluation_parameters.json b/...ns/column_map_expectations/expect_column_values_to_be_between__evaluation_parameters.json
@@ -163,7 +163,14 @@
           "title": "basic_negative_now_test_other_sql",
           "include_in_gallery": true,
           "exact_match_out": false,
-          "suppress_test_for": ["pandas", "spark", "mysql", "mssql", "postgresql"],
+          "suppress_test_for": [
+            "pandas",
+            "spark",
+            "mysql",
+            "mssql",
+            "postgresql",
+            "bigquery"
+          ],
           "out": {
             "unexpected_index_list": [{"pk_index": 0, "ts": "1970-01-01 12:00:01"}, {"pk_index": 1, "ts": "1999-12-31 12:00:01"}, {"pk_index": 2, "ts": "2000-01-01 12:00:01"}, {"pk_index": 3, "ts": "2000-02-01 12:00:01"}, {"pk_index": 4, "ts": "2000-03-01 12:00:01"}, {"pk_index": 5, "ts": "2000-04-01 12:00:01"}, {"pk_index": 6, "ts": "2000-05-01 12:00:01"}, {"pk_index": 7, "ts": "2000-06-01 12:00:01"}, {"pk_index": 9, "ts": "2001-01-01 12:00:01"}],
             "success": false
@@ -208,7 +215,14 @@
         {
           "title": "basic_negative_now_timedelta_test_other_sql",
           "exact_match_out": false,
-          "suppress_test_for": ["pandas", "spark", "postgresql", "mssql", "mysql"],
+          "suppress_test_for": [
+            "pandas",
+            "spark",
+            "postgresql",
+            "mssql",
+            "mysql",
+            "bigquery"
+          ],
           "out": {
             "unexpected_index_list": [{"pk_index": 0, "ts": "1970-01-01 12:00:01"}, {"pk_index": 1, "ts": "1999-12-31 12:00:01"}, {"pk_index": 2, "ts": "2000-01-01 12:00:01"}, {"pk_index": 3, "ts": "2000-02-01 12:00:01"}, {"pk_index": 4, "ts": "2000-03-01 12:00:01"}, {"pk_index": 5, "ts": "2000-04-01 12:00:01"}, {"pk_index": 6, "ts": "2000-05-01 12:00:01"}, {"pk_index": 7, "ts": "2000-06-01 12:00:01"}, {"pk_index": 9, "ts": "2001-01-01 12:00:01"}],
             "success": false
@@ -253,7 +267,14 @@
         }, {
           "title": "basic_negative_datetime_test_other_sql",
           "exact_match_out": false,
-          "suppress_test_for": ["pandas", "spark", "postgresql", "mssql", "mysql"],
+          "suppress_test_for": [
+            "pandas",
+            "spark",
+            "postgresql",
+            "mssql",
+            "mysql",
+            "bigquery"
+          ],
           "out": {
             "success": false,
             "unexpected_index_list": [{"pk_index": 0, "ts": "1970-01-01 12:00:01"}, {"pk_index": 1, "ts": "1999-12-31 12:00:01"}, {"pk_index": 2, "ts": "2000-01-01 12:00:01"}, {"pk_index": 3, "ts": "2000-02-01 12:00:01"}, {"pk_index": 4, "ts": "2000-03-01 12:00:01"}, {"pk_index": 5, "ts": "2000-04-01 12:00:01"}, {"pk_index": 6, "ts": "2000-05-01 12:00:01"}, {"pk_index": 7, "ts": "2000-06-01 12:00:01"}, {"pk_index": 9, "ts": "2001-01-01 12:00:01"}]

diff --git a/...t_definitions/column_map_expectations/expect_column_values_to_be_between_tz_informed.json b/...t_definitions/column_map_expectations/expect_column_values_to_be_between_tz_informed.json
@@ -137,7 +137,10 @@
       },
       {
         "title": "basic_negative_test_with_timestamps_tz_informed_sql",
-        "only_for": ["trino", "bigquery","sqlite"] ,
+        "only_for": [
+          "trino",
+          "sqlite"
+        ],
         "exact_match_out": false,
         "out": {
           "unexpected_index_list": [{"pk_index": 0, "ts": "1970-01-01 12:00:01.000000"}, {"pk_index": 8, "ts": "2019-09-01 15:27:05.345678"},

diff --git a/...test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json b/...test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json
@@ -256,7 +256,14 @@
       {
         "title": "basic_negative_test_with_timestamps_other_sql",
         "exact_match_out": false,
-        "suppress_test_for": ["pandas", "spark", "postgresql", "mssql", "mysql"],
+        "suppress_test_for": [
+          "pandas",
+          "spark",
+          "postgresql",
+          "mssql",
+          "mysql",
+          "bigquery"
+        ],
         "out": {
           "unexpected_index_list":  [{"ts": "1970-01-01 12:00:01", "pk_index": 0}, {"ts": "2001-01-01 12:00:01", "pk_index": 9}],
           "success": false

diff --git a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json
@@ -300,7 +300,12 @@
       },
       "out": {
         "success": false,
-        "unexpected_index_list": [0],
+        "unexpected_index_list": [
+          {
+            "pk_index": 0,
+            "dates": "2018-01-01"
+          }
+        ],
         "unexpected_list": ["2018-01-01"]
       }
     }

diff --git a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_unique.json b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_unique.json
@@ -158,7 +158,24 @@
         "out": {
           "success": false,
           "unexpected_list": ["None", "None", "None", "None"],
-          "unexpected_index_list": [0, 1, 2, 3]
+          "unexpected_index_list": [
+            {
+              "pk_index": 0,
+              "null": "None"
+            },
+            {
+              "pk_index": 1,
+              "null": "None"
+            },
+            {
+              "pk_index": 2,
+              "null": "None"
+            },
+            {
+              "pk_index": 3,
+              "null": "None"
+            }
+          ]
         }
       },
       {

diff --git a/tests/test_definitions/test_expectations_v3_api.py b/tests/test_definitions/test_expectations_v3_api.py
@@ -125,9 +125,6 @@ def pytest_generate_tests(metafunc):  # noqa C901 - 35
                                 validator_with_data.active_batch_data,
                                 SqlAlchemyBatchData,
                             ):
-                                # Call out supported dialects
-                                if "pandas_v3_api" in only_for:
-                                    generate_test = True
                                 if "sqlalchemy" in only_for:
                                     generate_test = True
                                 elif (
@@ -208,6 +205,9 @@ def pytest_generate_tests(metafunc):  # noqa C901 - 35
                                 validator_with_data.active_batch_data,
                                 PandasBatchData,
                             ):
+                                # Call out supported dialects
+                                if "pandas_v3_api" in only_for:
+                                    generate_test = True
                                 major, minor, *_ = pd.__version__.split(".")
                                 if "pandas" in only_for:
                                     generate_test = True