Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-50507][PYTHON][TESTS] Group pandas function related tests #49074

Closed
wants to merge 3 commits into from
Closed

[SPARK-50507][PYTHON][TESTS] Group pandas function related tests #49074

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7247233
fix
zhengruifeng Dec 4, 2024
b965632
fix
zhengruifeng Dec 4, 2024
e3d93d4
fix
zhengruifeng Dec 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions dev/sparktestsupport/modules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,8 +534,8 @@ def __hash__(self):
"pyspark.sql.tests.pandas.test_pandas_udf_typehints",
"pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations",
"pyspark.sql.tests.pandas.test_pandas_udf_window",
"pyspark.sql.tests.pandas.test_pandas_sqlmetrics",
"pyspark.sql.tests.pandas.test_converter",
"pyspark.sql.tests.test_pandas_sqlmetrics",
"pyspark.sql.tests.test_python_datasource",
"pyspark.sql.tests.test_python_streaming_datasource",
"pyspark.sql.tests.test_readwriter",
Expand Down Expand Up @@ -1054,11 +1054,7 @@ def __hash__(self):
"pyspark.sql.tests.connect.test_parity_memory_profiler",
"pyspark.sql.tests.connect.test_parity_udtf",
"pyspark.sql.tests.connect.test_parity_tvf",
"pyspark.sql.tests.connect.test_parity_pandas_udf",
"pyspark.sql.tests.connect.test_parity_pandas_map",
"pyspark.sql.tests.connect.test_parity_arrow_map",
"pyspark.sql.tests.connect.test_parity_pandas_grouped_map",
"pyspark.sql.tests.connect.test_parity_pandas_cogrouped_map",
"pyspark.sql.tests.connect.test_parity_arrow_grouped_map",
"pyspark.sql.tests.connect.test_parity_arrow_cogrouped_map",
"pyspark.sql.tests.connect.test_parity_python_datasource",
Expand All @@ -1074,13 +1070,17 @@ def __hash__(self):
"pyspark.sql.tests.connect.streaming.test_parity_listener",
"pyspark.sql.tests.connect.streaming.test_parity_foreach",
"pyspark.sql.tests.connect.streaming.test_parity_foreach_batch",
"pyspark.sql.tests.connect.test_parity_pandas_grouped_map_with_state",
"pyspark.sql.tests.connect.test_parity_pandas_udf_scalar",
"pyspark.sql.tests.connect.test_parity_pandas_udf_grouped_agg",
"pyspark.sql.tests.connect.test_parity_pandas_udf_window",
"pyspark.sql.tests.connect.test_resources",
"pyspark.sql.tests.connect.shell.test_progress",
"pyspark.sql.tests.connect.test_df_debug",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_map",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map_with_state",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_udf",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_scalar",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_grouped_agg",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_window",
],
excluded_python_implementations=[
"PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and
Expand Down
16 changes: 16 additions & 0 deletions python/pyspark/sql/tests/connect/pandas/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
2 changes: 1 addition & 1 deletion ...nnect/test_parity_pandas_cogrouped_map.py → ...andas/test_parity_pandas_cogrouped_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class CogroupedApplyInPandasTests(


if __name__ == "__main__":
from pyspark.sql.tests.connect.test_parity_pandas_cogrouped_map import * # noqa: F401
from pyspark.sql.tests.connect.pandas.test_parity_pandas_cogrouped_map import * # noqa: F401

try:
import xmlrunner
Expand Down
2 changes: 1 addition & 1 deletion ...connect/test_parity_pandas_grouped_map.py → .../pandas/test_parity_pandas_grouped_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def test_supported_types(self):


if __name__ == "__main__":
from pyspark.sql.tests.connect.test_parity_pandas_grouped_map import * # noqa: F401
from pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map import * # noqa: F401

try:
import xmlrunner
Expand Down
2 changes: 1 addition & 1 deletion ...t_parity_pandas_grouped_map_with_state.py → ...t_parity_pandas_grouped_map_with_state.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ class GroupedApplyInPandasWithStateTests(


if __name__ == "__main__":
from pyspark.sql.tests.connect.test_parity_pandas_grouped_map_with_state import * # noqa: F401
from pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map_with_state import * # noqa: F401,E501

try:
import xmlrunner
Expand Down
2 changes: 1 addition & 1 deletion ...l/tests/connect/test_parity_pandas_map.py → .../connect/pandas/test_parity_pandas_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class MapInPandasParityTests(

if __name__ == "__main__":
import unittest
from pyspark.sql.tests.connect.test_parity_pandas_map import * # noqa: F401
from pyspark.sql.tests.connect.pandas.test_parity_pandas_map import * # noqa: F401

try:
import xmlrunner # type: ignore[import]
Expand Down
2 changes: 1 addition & 1 deletion ...l/tests/connect/test_parity_pandas_udf.py → .../connect/pandas/test_parity_pandas_udf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class PandasUDFParityTests(PandasUDFTestsMixin, ReusedConnectTestCase):

if __name__ == "__main__":
import unittest
from pyspark.sql.tests.connect.test_parity_pandas_udf import * # noqa: F401
from pyspark.sql.tests.connect.pandas.test_parity_pandas_udf import * # noqa: F401

try:
import xmlrunner # type: ignore[import]
Expand Down
2 changes: 1 addition & 1 deletion ...ect/test_parity_pandas_udf_grouped_agg.py → ...das/test_parity_pandas_udf_grouped_agg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class PandasUDFGroupedAggParityTests(


if __name__ == "__main__":
from pyspark.sql.tests.connect.test_parity_pandas_udf_grouped_agg import * # noqa: F401
from pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_grouped_agg import * # noqa: F401

try:
import xmlrunner # type: ignore[import]
Expand Down
2 changes: 1 addition & 1 deletion .../connect/test_parity_pandas_udf_scalar.py → ...t/pandas/test_parity_pandas_udf_scalar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ class PandasUDFScalarParityTests(ScalarPandasUDFTestsMixin, ReusedConnectTestCas


if __name__ == "__main__":
from pyspark.sql.tests.connect.test_parity_pandas_udf_scalar import * # noqa: F401
from pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_scalar import * # noqa: F401

try:
import xmlrunner # type: ignore[import]
Expand Down
2 changes: 1 addition & 1 deletion .../connect/test_parity_pandas_udf_window.py → ...t/pandas/test_parity_pandas_udf_window.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class PandasUDFWindowParityTests(


if __name__ == "__main__":
from pyspark.sql.tests.connect.test_parity_pandas_udf_window import * # noqa: F401
from pyspark.sql.tests.connect.pandas.test_parity_pandas_udf_window import * # noqa: F401

try:
import xmlrunner # type: ignore[import]
Expand Down
2 changes: 1 addition & 1 deletion ...spark/sql/tests/test_pandas_sqlmetrics.py → ...ql/tests/pandas/test_pandas_sqlmetrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def test_pandas(col1):


if __name__ == "__main__":
from pyspark.sql.tests.test_pandas_sqlmetrics import * # noqa: F401
from pyspark.sql.tests.pandas.test_pandas_sqlmetrics import * # noqa: F401

try:
import xmlrunner
Expand Down
Loading