SNOW-742340 Add python 3.9 integ test merge gate (snowflakedb#681)

* SNOW-742340 Add python 3.9 integ test merge gate

Description

Add python 3.9 integ test non-required merge gate. These tests
explicitly skip udf test because 3.9 is not supported for UDFs yet

Testing

n/a

* passenv + pytest mark

* add py39 to coverage

* pass SNOWFLAKE_IS_PYTHON_RUNTIME_TEST to coverage

* more udf marks

* do not do doctest for py39

.github/workflows/precommit.yml

@@ -60,7 +60,7 @@ jobs:
             download_name: macos
           - image_name: windows-latest
             download_name: windows
-        python-version: [3.8]
+        python-version: [3.8, 3.9]
         cloud-provider: [aws, azure, gcp]
     steps:
       - name: Checkout Code
@@ -89,7 +89,7 @@ jobs:
         run: python -m pip install -U setuptools pip wheel
       - name: Install tox
         run: python -m pip install tox
-      - if: ${{ contains('macos', matrix.os.download_name) }}
+      - if: ${{ matrix.python-version == '3.8' && contains('macos', matrix.os.download_name) }}
         name: Run tests (including doctest Tests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-udf"
         env:
@@ -98,7 +98,7 @@ jobs:
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
-      - if: ${{ false == contains('macos', matrix.os.download_name) }}
+      - if: ${{ matrix.python-version == '3.8' && false == contains('macos', matrix.os.download_name) }}
         name: Run tests (excluding doctest Tests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest"
         env:
@@ -107,9 +107,21 @@ jobs:
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
+      - if: ${{ matrix.python-version == '3.9' }}
+        name: Run tests (including doctest Tests, excluding udf tests)
+        run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
+        shell: bash
       - name: Combine coverages
         run: python -m tox -e coverage --skip-missing-interpreters false
         shell: bash
+        env:
+          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
       - uses: actions/upload-artifact@v2
         with:
           name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}

setup.py

@@ -11,6 +11,9 @@
 SRC_DIR = os.path.join(THIS_DIR, "src")
 SNOWPARK_SRC_DIR = os.path.join(SRC_DIR, "snowflake", "snowpark")
 CONNECTOR_DEPENDENCY_VERSION = ">=2.7.12, <4.0.0"
+REQUIRED_PYTHON_VERSION = "==3.8.*"
+if os.getenv("SNOWFLAKE_IS_PYTHON_RUNTIME_TEST", False):
+    REQUIRED_PYTHON_VERSION = ">=3.8"
 
 # read the version
 VERSION = ()
@@ -42,7 +45,7 @@
         "Issues": "https://github.com/snowflakedb/snowpark-python/issues",
         "Changelog": "https://github.com/snowflakedb/snowpark-python/blob/main/CHANGELOG.md",
     },
-    python_requires="==3.8.*",
+    python_requires=REQUIRED_PYTHON_VERSION,
     install_requires=[
         "setuptools>=40.6.0",
         "wheel",

tests/integ/scala/test_udtf_suite.py

@@ -24,6 +24,8 @@
 )
 from tests.utils import Utils
 
+pytestmark = pytest.mark.udf
+
 wordcount_table_name = Utils.random_table_name()
 
 

tests/integ/test_column_names.py

@@ -345,6 +345,7 @@ def test_function_expression(session):
     )
 
 
+@pytest.mark.udf
 @pytest.mark.parametrize("use_qualified_name", [True, False])
 def test_udf(session, use_qualified_name):
     def add_one(x: int) -> int:

tests/integ/test_dataframe.py

@@ -337,6 +337,7 @@ def test_select_star(session):
     assert res == expected
 
 
+@pytest.mark.udf
 def test_select_table_function(session):
     df = session.create_dataframe(
         [(1, "one o one", 10), (2, "twenty two", 20), (3, "thirty three", 30)]
@@ -518,6 +519,7 @@ def test_generator_table_function_negative(session):
     assert "Columns cannot be empty for generator table function" in str(ex_info)
 
 
+@pytest.mark.udf
 def test_select_table_function_negative(session):
     df = session.create_dataframe([(1, "a", 10), (2, "b", 20), (3, "c", 30)]).to_df(
         ["a", "b", "c"]
@@ -554,6 +556,7 @@ def process(self, n: int) -> Iterable[Tuple[int, int]]:
     )
 
 
+@pytest.mark.udf
 def test_with_column(session):
     df = session.create_dataframe([[1, 2], [3, 4]], schema=["a", "b"])
     expected = [Row(A=1, B=2, MEAN=1.5), Row(A=3, B=4, MEAN=3.5)]
@@ -568,6 +571,7 @@ def process(self, a: int, b: int) -> Iterable[Tuple[int]]:
     Utils.check_answer(df.with_column("total", sum_udtf(df.a, df.b)), expected)
 
 
+@pytest.mark.udf
 def test_with_column_negative(session):
     df = session.create_dataframe([[1, 2], [3, 4]], schema=["a", "b"])
 
@@ -585,6 +589,7 @@ def process(self, a: int, b: int) -> Iterable[Tuple[int, int]]:
     )
 
 
+@pytest.mark.udf
 def test_with_columns(session):
     df = session.create_dataframe([[1, 2], [3, 4]], schema=["a", "b"])
 
@@ -655,6 +660,7 @@ def process(self, a: int, b: int) -> Iterable[Tuple[int, int]]:
     )
 
 
+@pytest.mark.udf
 def test_with_columns_negative(session):
     df = session.create_dataframe(
         [[1, 2, "one o one"], [3, 4, "two o two"]], schema=["a", "b", "c"]

tests/integ/test_simplifier_suite.py

@@ -360,6 +360,7 @@ def test_select_expr(session, simplifier_table):
     assert df11.queries["queries"][-1].count("SELECT") == 2
 
 
+@pytest.mark.udf
 def test_select_with_table_function_join(session):
     # setup
     df = session.create_dataframe(
@@ -431,6 +432,7 @@ def process(self, n: int) -> Iterable[Tuple[int, int]]:
     )
 
 
+@pytest.mark.udf
 def test_join_table_function(session):
     # setup
     df = session.create_dataframe(

tests/integ/test_stored_procedure.py

@@ -33,6 +33,8 @@
 )
 from tests.utils import IS_IN_STORED_PROC, TempObjectType, TestFiles, Utils
 
+pytestmark = pytest.mark.udf
+
 try:
     import numpy  # noqa: F401
     import pandas  # noqa: F401

tests/integ/test_telemetry.py

@@ -744,6 +744,7 @@ def test_dataframe_na_functions_api_calls(session):
     assert df2._plan.api_calls == [{"name": "Session.sql"}]
 
 
+@pytest.mark.udf
 def test_udf_call_and_invoke(session, resources_path):
     telemetry_tracker = TelemetryDataTracker(session)
     df = session.create_dataframe([[1, 2]], schema=["a", "b"])
@@ -821,6 +822,7 @@ def add_one_df_pandas_udf(df):
     assert data == {"func_name": "functions.call_udf", "category": "usage"}
 
 
+@pytest.mark.udf
 def test_sproc_call_and_invoke(session, resources_path):
     telemetry_tracker = TelemetryDataTracker(session)
 
@@ -870,6 +872,7 @@ def add_one(session_, x):
     assert data == {"func_name": "StoredProcedure.__call__", "category": "usage"}
 
 
+@pytest.mark.udf
 def test_udtf_call_and_invoke(session, resources_path):
     telemetry_tracker = TelemetryDataTracker(session)
     df = session.create_dataframe([[1, 2]], schema=["a", "b"])

tests/integ/test_udtf.py

@@ -20,6 +20,8 @@
 )
 from tests.utils import TestFiles, Utils
 
+pytestmark = pytest.mark.udf
+
 
 def test_register_udtf_from_file_no_type_hints(session, resources_path):
     test_files = TestFiles(resources_path)

tox.ini

@@ -55,23 +55,27 @@ passenv =
     USERNAME
     CLIENT_LOG_DIR_PATH_DOCKER
     PYTEST_ADDOPTS
+    SNOWFLAKE_IS_PYTHON_RUNTIME_TEST
 commands =
     notudf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} src/snowflake/snowpark tests
     udf: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} src/snowflake/snowpark tests
     notdoctest: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} tests
+    notudfdoctest: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} tests
 
 [testenv:coverage]
 description = [run locally after tests]: combine coverage data and create report
 deps = {[testenv]deps}
        coverage
 skip_install = True
-passenv = DIFF_AGAINST
+passenv =
+    DIFF_AGAINST
+    SNOWFLAKE_IS_PYTHON_RUNTIME_TEST
 setenv = COVERAGE_FILE={toxworkdir}/.coverage
 commands = coverage combine
            coverage report -m
            coverage xml -o {env:COV_REPORT_DIR:{toxworkdir}}/coverage.xml
            coverage html -d {env:COV_REPORT_DIR:{toxworkdir}}/htmlcov
-depends = py38
+depends = py38, py39
 
 [testenv:flake8]
 description = check code style with flake8