Add Python3.8 to TravisCI and no Python27 (#192)

* Add Python3.8 to TravisCI and no Python27
* Install pyspark version supporting Python38

flytekit/.travis.yml

@@ -1,6 +1,10 @@
 language: python
 jobs:
   include:
+    - name: "Python 3.8"
+      python: 3.8
+      before_install:
+        - make setup-spark3
     - name: "Python 3.7"
       python: 3.7
       before_install:

flytekit/Makefile

@@ -17,6 +17,10 @@ install-piptools:
 setup: install-piptools ## Install requirements
 	pip-sync requirements.txt dev-requirements.txt
 
+.PHONY: setup-spark3
+setup-spark3: install-piptools ## Install requirements
+	pip-sync requirements-spark3.txt dev-requirements.txt
+
 .PHONY: fmt
 fmt: ## Format code with black and isort
 	black .

flytekit/dev-requirements.txt

@@ -26,7 +26,7 @@ pycodestyle==2.6.0        # via flake8
 pyflakes==2.2.0           # via flake8
 pyparsing==2.4.7          # via packaging
 pytest==6.0.2             # via -r dev-requirements.in
-regex==2020.7.14          # via -c requirements.txt, black
+regex==2020.9.27          # via -c requirements.txt, black
 six==1.15.0               # via -c requirements.txt, packaging
 testfixtures==6.14.2      # via flake8-isort
 toml==0.10.1              # via -c requirements.txt, black, pytest

flytekit/flytekit/__init__.py

@@ -2,7 +2,7 @@
 
 import flytekit.plugins  # noqa: F401
 
-__version__ = "0.13.0b10"
+__version__ = "0.13.0b11"
 
 logger = _logging.getLogger("flytekit")
 

flytekit/requirements-spark3.in

@@ -0,0 +1,2 @@
+.[all-spark3]
+-e file:.#egg=flytekit

flytekit/requirements-spark3.txt

@@ -0,0 +1,88 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile requirements-spark3.in
+#
+-e file:.#egg=flytekit    # via -r requirements-spark3.in
+ansiwrap==0.8.4           # via papermill
+appdirs==1.4.4            # via black
+async-generator==1.10     # via nbclient
+attrs==20.2.0             # via black, jsonschema
+backcall==0.2.0           # via ipython
+black==19.10b0            # via flytekit, papermill
+boto3==1.15.11            # via flytekit
+botocore==1.18.11         # via boto3, s3transfer
+certifi==2020.6.20        # via requests
+cffi==1.14.3              # via cryptography
+chardet==3.0.4            # via requests
+click==7.1.2              # via black, flytekit, hmsclient, papermill
+croniter==0.3.34          # via flytekit
+cryptography==3.1.1       # via secretstorage
+decorator==4.4.2          # via ipython
+deprecated==1.2.10        # via flytekit
+entrypoints==0.3          # via papermill
+flyteidl==0.18.8          # via flytekit
+future==0.18.2            # via torch
+grpcio==1.32.0            # via flytekit
+hmsclient==0.1.1          # via flytekit
+idna==2.10                # via requests
+ipykernel==5.3.4          # via flytekit
+ipython-genutils==0.2.0   # via nbformat, traitlets
+ipython==7.18.1           # via ipykernel
+jedi==0.17.2              # via ipython
+jeepney==0.4.3            # via keyring, secretstorage
+jmespath==0.10.0          # via boto3, botocore
+jsonschema==3.2.0         # via nbformat
+jupyter-client==6.1.7     # via ipykernel, nbclient
+jupyter-core==4.6.3       # via jupyter-client, nbformat
+k8s-proto==0.0.3          # via flytekit
+keyring==21.4.0           # via flytekit
+natsort==7.0.1            # via croniter
+nbclient==0.5.0           # via papermill
+nbformat==5.0.7           # via nbclient, papermill
+nest-asyncio==1.4.1       # via nbclient
+numpy==1.19.2             # via flytekit, pandas, pyarrow, torch
+pandas==1.1.2             # via flytekit
+papermill==2.2.0          # via flytekit
+parso==0.7.1              # via jedi
+pathspec==0.8.0           # via black
+pexpect==4.8.0            # via ipython
+pickleshare==0.7.5        # via ipython
+prompt-toolkit==3.0.7     # via ipython
+protobuf==3.13.0          # via flyteidl, flytekit, k8s-proto
+ptyprocess==0.6.0         # via pexpect
+py4j==0.10.9              # via pyspark
+pyarrow==0.17.1           # via flytekit
+pycparser==2.20           # via cffi
+pygments==2.7.1           # via ipython
+pyrsistent==0.17.3        # via jsonschema
+pyspark==3.0.1            # via flytekit
+python-dateutil==2.8.1    # via botocore, croniter, flytekit, jupyter-client, pandas
+pytimeparse==1.1.8        # via flytekit
+pytz==2018.4              # via flytekit, pandas
+pyyaml==5.3.1             # via papermill
+pyzmq==19.0.2             # via jupyter-client
+regex==2020.9.27          # via black
+requests==2.24.0          # via flytekit, papermill, responses
+responses==0.12.0         # via flytekit
+s3transfer==0.3.3         # via boto3
+secretstorage==3.1.2      # via keyring
+six==1.15.0               # via cryptography, flytekit, grpcio, jsonschema, protobuf, python-dateutil, responses, tenacity, thrift
+sortedcontainers==2.2.2   # via flytekit
+statsd==3.3.0             # via flytekit
+tenacity==6.2.0           # via papermill
+textwrap3==0.9.2          # via ansiwrap
+thrift==0.13.0            # via hmsclient
+toml==0.10.1              # via black
+torch==1.6.0              # via flytekit
+tornado==6.0.4            # via ipykernel, jupyter-client
+tqdm==4.50.0              # via papermill
+traitlets==5.0.4          # via ipykernel, ipython, jupyter-client, jupyter-core, nbclient, nbformat
+typed-ast==1.4.1          # via black
+urllib3==1.25.10          # via botocore, flytekit, requests, responses
+wcwidth==0.2.5            # via prompt-toolkit
+wrapt==1.12.1             # via deprecated, flytekit
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

flytekit/requirements.txt

@@ -61,7 +61,7 @@ pytimeparse==1.1.8        # via flytekit
 pytz==2018.4              # via flytekit, pandas
 pyyaml==5.3.1             # via papermill
 pyzmq==19.0.2             # via jupyter-client
-regex==2020.7.14          # via black
+regex==2020.9.27          # via black
 requests==2.24.0          # via flytekit, papermill, responses
 responses==0.12.0         # via flytekit
 s3transfer==0.3.3         # via boto3

flytekit/setup.py

@@ -5,12 +5,6 @@
 
 extras_require = LazyLoadPlugin.get_extras_require()
 
-extras_require[':python_version<"3"'] = [
-    "configparser>=3.0.0,<4.0.0",
-    "futures>=3.2.0,<4.0.0",
-    "pathlib2>=2.3.2,<3.0.0",
-]
-
 setup(
     name="flytekit",
     version=flytekit.__version__,
@@ -65,7 +59,6 @@
         "Intended Audience :: Developers",
         "Intended Audience :: System Administrators",
         "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",