Merge branch 'master' into feat/security-converge-databases

# Conflicts:
#	superset/constants.py
#	tests/security_tests.py

.github/workflows/superset-e2e.yml

@@ -1,13 +1,18 @@
 name: E2E
 
-on: [push, pull_request]
+on: [push, pull_request_target]
 
 jobs:
-  Cypress:
+  cypress-matrix:
     runs-on: ubuntu-18.04
     strategy:
-      fail-fast: true
+      # when one test fails, DO NOT cancel the other
+      # containers, because this will kill Cypress processes
+      # leaving the Dashboard hanging ...
+      # https://github.com/cypress-io/github-action/issues/48
+      fail-fast: false
       matrix:
+        containers: [1, 2, 3]
         browser: ["chrome"]
     env:
       FLASK_ENV: development
@@ -34,8 +39,14 @@ jobs:
         uses: styfle/[email protected]
         with:
           access_token: ${{ github.token }}
-      - name: Checkout code
+      - name: Checkout code (push)
+        if: github.event_name == 'push'
+        uses: actions/checkout@v2
+      - name: Checkout code (pull_request)
+        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
         uses: actions/checkout@v2
+        with:
+          ref: 'refs/pull/${{ github.event.number }}/merge'
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
@@ -89,3 +100,12 @@ jobs:
         with:
           name: screenshots
           path: ${{ github.workspace }}/superset-frontend/cypress-base/cypress/screenshots
+  Cypress:
+    if: ${{ always() }}
+    name: Cypress (chrome)
+    runs-on: ubuntu-18.04
+    needs: cypress-matrix
+    steps:
+      - name: Check build matrix status
+        if: ${{ needs.cypress-matrix.result != 'success' }}
+        run: exit 1

CONTRIBUTING.md

@@ -672,6 +672,28 @@ CYPRESS_BASE_URL=<your url> npm run cypress open
 
 See [`superset-frontend/cypress_build.sh`](https://github.com/apache/incubator-superset/blob/master/superset-frontend/cypress_build.sh).
 
+As an alternative you can use docker-compose environment for testing:
+
+Make sure you have added below line to your /etc/hosts file:
+```127.0.0.1 db```
+
+If you already have launched Docker environment please use the following command to assure a fresh database instance:
+```docker-compose down -v```
+
+Launch environment:
+
+```CYPRESS_CONFIG=true docker-compose up```
+
+It will serve backend and frontend on port 8088.
+
+Run Cypress tests:
+
+```bash
+cd cypress-base
+npm install
+npm run cypress open
+```
+
 ### Storybook
 
 Superset includes a [Storybook](https://storybook.js.org/) to preview the layout/styling of various Superset components, and variations thereof. To open and view the Storybook:

docker-compose.yml

@@ -24,6 +24,7 @@ x-superset-volumes: &superset-volumes
   - ./superset:/app/superset
   - ./superset-frontend:/app/superset-frontend
   - superset_home:/app/superset_home
+  - ./tests:/app/tests
 
 version: "3.7"
 services:
@@ -57,6 +58,8 @@ services:
     user: "root"
     depends_on: *superset-depends-on
     volumes: *superset-volumes
+    environment:
+      CYPRESS_CONFIG: "${CYPRESS_CONFIG}"
 
   superset-init:
     image: *superset-image
@@ -66,6 +69,8 @@ services:
     depends_on: *superset-depends-on
     user: "root"
     volumes: *superset-volumes
+    environment:
+      CYPRESS_CONFIG: "${CYPRESS_CONFIG}"
 
   superset-node:
     image: node:12

docker/.env

@@ -42,3 +42,4 @@ REDIS_PORT=6379
 FLASK_ENV=development
 SUPERSET_ENV=development
 SUPERSET_LOAD_EXAMPLES=yes
+CYPRESS_CONFIG=false

docker/docker-bootstrap.sh

@@ -19,7 +19,13 @@
 set -eo pipefail
 
 REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
-
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    export SUPERSET_CONFIG=tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export ENABLE_REACT_CRUD_VIEWS=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
 #
 # Make sure we have dev requirements installed
 #

docker/docker-init.sh

@@ -37,22 +37,29 @@ Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
 
 EOF
 }
-
+ADMIN_PASSWORD="admin"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    ADMIN_PASSWORD="general"
+    export SUPERSET_CONFIG=tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export ENABLE_REACT_CRUD_VIEWS=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
 # Initialize the database
 echo_step "1" "Starting" "Applying DB migrations"
 superset db upgrade
 echo_step "1" "Complete" "Applying DB migrations"
 
 # Create an admin user
-echo_step "2" "Starting" "Setting up admin user ( admin / admin )"
+echo_step "2" "Starting" "Setting up admin user ( admin / $ADMIN_PASSWORD )"
 superset fab create-admin \
               --username admin \
               --firstname Superset \
               --lastname Admin \
               --email [email protected] \
-              --password admin
+              --password $ADMIN_PASSWORD
 echo_step "2" "Complete" "Setting up admin user"
-
 # Create default roles and permissions
 echo_step "3" "Starting" "Setting up roles and perms"
 superset init
@@ -61,6 +68,12 @@ echo_step "3" "Complete" "Setting up roles and perms"
 if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
     # Load some data to play with
     echo_step "4" "Starting" "Loading examples"
-    superset load_examples
+    # If Cypress run which consumes superset_test_config – load required data for tests
+    if [ "$CYPRESS_CONFIG" == "true" ]; then
+        superset load_test_users
+        superset load_examples --load-test-data
+    else
+        superset load_examples
+    fi
     echo_step "4" "Complete" "Loading examples"
 fi

docs/src/pages/docs/Miscellaneous/issue_codes.mdx

@@ -73,3 +73,14 @@ The table was deleted or renamed in the database.
 Your query failed because it is referencing a table that no longer exists in
 the underlying database. You should modify your query to reference the correct
 table.
+
+## Issue 1006
+
+```
+One or more parameters specified in the query are missing.
+```
+
+Your query was not submitted to the database because it's missing one or more
+parameters. You should define all the parameters referenced in the query in a
+valid JSON document. Check that the parameters are spelled correctly and that
+the document has a valid syntax.

requirements/base.in

@@ -17,4 +17,4 @@
 # under the License.
 -e file:.
 pyrsistent>=0.16.1,<0.17
-importlib-metadata>=1.7.0, <2
+importlib-metadata==2.1.1

requirements/base.txt

@@ -1,4 +1,4 @@
-# SHA1:1b5e15727cda5c8d6f711f02629430e1560618eb
+# SHA1:9eeea842b4dffc85a3611351dcb33aca43fbb9e9
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -24,6 +24,7 @@ chardet==3.0.4            # via aiohttp
 click==7.1.2              # via apache-superset, flask, flask-appbuilder
 colorama==0.4.4           # via apache-superset, flask-appbuilder
 contextlib2==0.6.0.post1  # via apache-superset
+cron-descriptor==1.2.24   # via apache-superset
 croniter==0.3.36          # via apache-superset
 cryptography==3.2.1       # via apache-superset
 decorator==4.4.2          # via retry
@@ -47,7 +48,7 @@ geopy==2.0.0              # via apache-superset
 gunicorn==20.0.4          # via apache-superset
 humanize==3.1.0           # via apache-superset
 idna==2.10                # via email-validator, yarl
-importlib-metadata==1.7.0  # via -r requirements/base.in
+importlib-metadata==2.1.1  # via -r requirements/base.in, jsonschema, kombu, markdown
 isodate==0.6.0            # via apache-superset
 itsdangerous==1.1.0       # via flask, flask-wtf
 jinja2==2.11.2            # via flask, flask-babel
@@ -83,6 +84,7 @@ python-geohash==0.8.5     # via apache-superset
 python3-openid==3.2.0     # via flask-openid
 pytz==2020.4              # via babel, celery, flask-babel, pandas
 pyyaml==5.3.1             # via apache-superset, apispec
+redis==3.5.3              # via apache-superset
 retry==0.9.2              # via apache-superset
 selenium==3.141.0         # via apache-superset
 simplejson==3.17.2        # via apache-superset
@@ -91,7 +93,7 @@ slackclient==2.5.0        # via apache-superset
 sqlalchemy-utils==0.36.8  # via apache-superset, flask-appbuilder
 sqlalchemy==1.3.20        # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 sqlparse==0.3.0           # via apache-superset
-typing-extensions==3.7.4.3  # via aiohttp
+typing-extensions==3.7.4.3  # via aiohttp, yarl
 urllib3==1.25.11          # via selenium
 vine==1.3.0               # via amqp, celery
 webencodings==0.5.1       # via bleach

requirements/development.in

@@ -17,10 +17,10 @@
 # under the License.
 -r base.in
 flask-cors>=2.0.0
+mysqlclient==1.4.2.post1
+pillow>=7.0.0,<8.0.0
 pydruid>=0.6.1,<0.7
 pyhive[hive]>=0.6.1
-tableschema
-thrift>=0.11.0, <1.0.0
-mysqlclient==1.4.2.post1
 psycopg2-binary==2.8.5
-Pillow>=7.0.0, <8.0.0
+tableschema
+thrift>=0.11.0,<1.0.0

requirements/development.txt

@@ -1,4 +1,4 @@
-# SHA1:3c2ff5fb241e76e4d027742510de3b719d39e0f2
+# SHA1:ae0364cae066a5cb8fb543c4f568bfcdacb6c1b7
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:

requirements/docker.in

@@ -17,4 +17,3 @@
 -r base.in
 gevent
 psycopg2-binary
-redis

requirements/docker.txt

@@ -1,4 +1,4 @@
-# SHA1:54074dcfe0a3bf83a9f4f8e2fb9d24746a86b50c
+# SHA1:465a3e60283551f04dc2897dc9f08d28b0d85228
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -10,7 +10,6 @@
 gevent==20.9.0            # via -r requirements/docker.in
 greenlet==0.4.17          # via gevent
 psycopg2-binary==2.8.6    # via -r requirements/docker.in
-redis==3.5.3              # via -r requirements/docker.in
 zope.event==4.5.0         # via gevent
 zope.interface==5.1.2     # via gevent
 

requirements/integration.txt

@@ -11,6 +11,7 @@ click==7.1.2              # via pip-compile-multi, pip-tools
 distlib==0.3.1            # via virtualenv
 filelock==3.0.12          # via tox, virtualenv
 identify==1.5.9           # via pre-commit
+importlib-metadata==2.1.1  # via pluggy, pre-commit, tox, virtualenv
 nodeenv==1.5.0            # via pre-commit
 packaging==20.4           # via tox
 pip-compile-multi==2.1.0  # via -r requirements/integration.in
@@ -25,6 +26,7 @@ toml==0.10.2              # via pre-commit, tox
 toposort==1.5             # via pip-compile-multi
 tox==3.20.1               # via -r requirements/integration.in
 virtualenv==20.1.0        # via pre-commit, tox
+zipp==3.4.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

requirements/testing.in

@@ -30,5 +30,4 @@ pyhive[presto]>=0.6.3
 pylint
 pytest
 pytest-cov
-redis
 statsd

requirements/testing.txt

@@ -1,4 +1,4 @@
-# SHA1:b16b83f856b2dbc53535f71414f5c3e8dfa838e0
+# SHA1:9d449781bc4ef88cd346b9dd5db55240472d5f0c
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -35,9 +35,9 @@ pyhive[hive,presto]==0.6.3  # via -r requirements/development.in, -r requirement
 pylint==2.6.0             # via -r requirements/testing.in
 pytest-cov==2.10.1        # via -r requirements/testing.in
 pytest==6.1.2             # via -r requirements/testing.in, pytest-cov
-redis==3.5.3              # via -r requirements/testing.in
 statsd==3.3.0             # via -r requirements/testing.in
 traitlets==5.0.5          # via ipython
+typed-ast==1.4.1          # via astroid
 wcwidth==0.2.5            # via prompt-toolkit
 websocket-client==0.57.0  # via docker
 wrapt==1.12.1             # via astroid

setup.cfg

@@ -30,7 +30,7 @@ combine_as_imports = true
 include_trailing_comma = true
 line_length = 88
 known_first_party = superset
-known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,contextlib2,croniter,cryptography,dateutil,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,msgpack,numpy,pandas,parameterized,parsedatetime,pathlib2,pgsanity,polyline,prison,pyarrow,pyhive,pytest,pytz,redis,retry,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,werkzeug,wtforms,wtforms_json,yaml
+known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,contextlib2,cron_descriptor,croniter,cryptography,dateutil,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,msgpack,numpy,pandas,parameterized,parsedatetime,pathlib2,pgsanity,pkg_resources,polyline,prison,pyarrow,pyhive,pytest,pytz,redis,retry,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,werkzeug,wtforms,wtforms_json,yaml
 multi_line_output = 3
 order_by_type = false
 

setup.py

@@ -72,6 +72,7 @@ def get_git_sha():
         "colorama",
         "contextlib2",
         "croniter>=0.3.28",
+        "cron-descriptor",
         "cryptography>=3.2.1",
         "flask>=1.1.0, <2.0.0",
         "flask-appbuilder>=3.1.1, <4.0.0",
@@ -96,6 +97,7 @@ def get_git_sha():
         "python-geohash",
         "pyarrow>=1.0.1, <1.1",
         "pyyaml>=5.1",
+        "redis",
         "retry>=0.9.2",
         "selenium>=3.141.0",
         "simplejson>=3.15.0",