From 8850376cbe48c6588b80bd4baa54fd7ad9384ec8 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Wed, 21 Apr 2021 18:12:39 +0100 Subject: [PATCH 1/6] feat: add alerts & reports to docker compose --- Dockerfile | 22 +++++++++++++++ docker/pythonpath_dev/superset_config.py | 36 ++++++++++++++++++++---- superset/config.py | 3 ++ superset/reports/commands/execute.py | 14 +++++++-- 4 files changed, 67 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index bbc8640d4ed55..d2a73f26b49d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,10 +123,32 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] # Dev image... ###################################################################### FROM lean AS dev +ARG CHROMEDRIVER_VERSION=84.0.4147.30 +ARG CHROME_VERSION=84.0.4147.105-1 COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/ USER root + +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends libnss3 + +# Install Chrome WebDriver +RUN mkdir -p /opt/chromedriver-${CHROMEDRIVER_VERSION} && \ + wget http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip -O /tmp/chromedriver_linux64.zip && \ + unzip /tmp/chromedriver_linux64.zip -d /opt/chromedriver-$CHROMEDRIVER_VERSION && \ + rm /tmp/chromedriver_linux64.zip && \ + chmod +x /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver && \ + ln -fs /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver /usr/local/bin/chromedriver + +# Install Google Chrome +RUN wget http://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${CHROME_VERSION}_amd64.deb -O /tmp/chrome.deb \ + && apt-get -y install /tmp/chrome.deb \ + # Make sure libzstd1 is latest (ID'd as potential vulnerability by Snyk) \ + && apt-get upgrade -y libzstd1 \ + && rm /tmp/chrome.deb + + # Cache everything for dev purposes... RUN cd /app \ && pip install --no-cache -r requirements/docker.txt \ diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index c88f3ab991e8e..eefde59b27b72 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -20,11 +20,12 @@ # development environments. Also note that superset_config_docker.py is imported # as a final step as a means to override "defaults" configured here # - import logging import os +from datetime import timedelta from cachelib.file import FileSystemCache +from celery.schedules import crontab logger = logging.getLogger() @@ -70,15 +71,40 @@ def get_env_variable(var_name, default=None): class CeleryConfig(object): BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}" - CELERY_IMPORTS = ("superset.sql_lab",) + CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks") CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}" - CELERY_ANNOTATIONS = {"tasks.add": {"rate_limit": "10/s"}} - CELERY_TASK_PROTOCOL = 1 - + CELERYD_LOG_LEVEL = "DEBUG" + CELERYD_PREFETCH_MULTIPLIER = 1 + CELERY_ACKS_LATE = False + CELERYBEAT_SCHEDULE = { + "reports.scheduler": { + "task": "reports.scheduler", + "schedule": crontab(minute="*", hour="*"), + }, + "reports.prune_log": { + "task": "reports.prune_log", + "schedule": crontab(minute=10, hour=0), + }, + } + + +WEBDRIVER_BASEURL = "http://superset:8088/" +# The base URL for the email report hyperlinks. +WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL +WEBDRIVER_TYPE = "chrome" +WEBDRIVER_OPTION_ARGS = [ + "--headless", + "--no-sandbox", + "--disable-dev-shm-usage", + "--disable-crash-reporter", +] +ALERT_REPORTS_NOTIFICATION_DRY_RUN = True CELERY_CONFIG = CeleryConfig SQLLAB_CTAS_NO_LIMIT = True +FEATURE_FLAGS = {"ALERT_REPORTS": True} + # # Optionally import superset_config_docker.py (which will have been included on # the PYTHONPATH) in order to allow for local settings to be overridden diff --git a/superset/config.py b/superset/config.py index 3511d67d192d6..2a270557132d0 100644 --- a/superset/config.py +++ b/superset/config.py @@ -956,6 +956,9 @@ class CeleryConfig: # pylint: disable=too-few-public-methods # if ALERT_REPORTS_WORKING_TIME_OUT_KILL is True, set a celery hard timeout # Equal to working timeout + ALERT_REPORTS_WORKING_SOFT_TIME_OUT_LAG ALERT_REPORTS_WORKING_SOFT_TIME_OUT_LAG = 1 +# If set to true no notification is sent, the worker will just log a message. +# Useful for debugging +ALERT_REPORTS_NOTIFICATION_DRY_RUN = False # A custom prefix to use on all Alerts & Reports emails EMAIL_REPORTS_SUBJECT_PREFIX = "[Report] " diff --git a/superset/reports/commands/execute.py b/superset/reports/commands/execute.py index befd0b2332ce1..cc7222e30290b 100644 --- a/superset/reports/commands/execute.py +++ b/superset/reports/commands/execute.py @@ -270,9 +270,10 @@ def _get_notification_content(self) -> NotificationContent: csv=csv_data, ) - @staticmethod def _send( - notification_content: NotificationContent, recipients: List[ReportRecipients] + self, + notification_content: NotificationContent, + recipients: List[ReportRecipients], ) -> None: """ Sends a notification to all recipients @@ -283,7 +284,14 @@ def _send( for recipient in recipients: notification = create_notification(recipient, notification_content) try: - notification.send() + if app.config["ALERT_REPORTS_NOTIFICATION_DRY_RUN"]: + logger.info( + "Would send notification for alert %s, to %s", + self._report_schedule.name, + recipient.recipient_config_json, + ) + else: + notification.send() except NotificationError as ex: # collect notification errors but keep processing them notification_errors.append(str(ex)) From d7fd1d65eefd0a5909979bf12f0473c7a0d16800 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Thu, 22 Apr 2021 11:14:43 +0100 Subject: [PATCH 2/6] add test --- tests/reports/commands_tests.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/reports/commands_tests.py b/tests/reports/commands_tests.py index 8392d74170839..bc0bf792a7b2d 100644 --- a/tests/reports/commands_tests.py +++ b/tests/reports/commands_tests.py @@ -568,7 +568,7 @@ def test_email_chart_report_schedule( screenshot_mock, email_mock, create_report_email_chart, ): """ - ExecuteReport Command: Test chart email report schedule with + ExecuteReport Command: Test chart email report schedule with screenshot """ # setup screenshot mock screenshot_mock.return_value = SCREENSHOT_FILE @@ -596,6 +596,29 @@ def test_email_chart_report_schedule( assert_log(ReportState.SUCCESS) +@pytest.mark.usefixtures( + "load_birth_names_dashboard_with_slices", "create_report_email_chart" +) +@patch("superset.reports.notifications.email.send_email_smtp") +@patch("superset.utils.screenshots.ChartScreenshot.get_screenshot") +def test_email_chart_report_dry_run( + screenshot_mock, email_mock, create_report_email_chart, +): + """ + ExecuteReport Command: Test chart email report schedule dry run + """ + # setup screenshot mock + screenshot_mock.return_value = SCREENSHOT_FILE + app.config["ALERT_REPORTS_NOTIFICATION_DRY_RUN"] = True + with freeze_time("2020-01-01T00:00:00Z"): + AsyncExecuteReportScheduleCommand( + TEST_ID, create_report_email_chart.id, datetime.utcnow() + ).run() + + email_mock.assert_not_called() + app.config["ALERT_REPORTS_NOTIFICATION_DRY_RUN"] = False + + @pytest.mark.usefixtures( "load_birth_names_dashboard_with_slices", "create_report_email_chart_with_csv" ) From 1e5f2004186f7b7c3ebdd1eabca5bf9576a15db7 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Thu, 22 Apr 2021 15:08:49 +0100 Subject: [PATCH 3/6] change to firefox --- Dockerfile | 33 ++++++++++-------------- docker/pythonpath_dev/superset_config.py | 10 +------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index d2a73f26b49d5..9bee98a90c334 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,31 +123,26 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] # Dev image... ###################################################################### FROM lean AS dev -ARG CHROMEDRIVER_VERSION=84.0.4147.30 -ARG CHROME_VERSION=84.0.4147.105-1 +ARG GECKODRIVER_VERSION=v0.28.0 +ARG FIREFOX_VERSION=88.0 COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/ USER root RUN apt-get update -y \ - && apt-get install -y --no-install-recommends libnss3 - -# Install Chrome WebDriver -RUN mkdir -p /opt/chromedriver-${CHROMEDRIVER_VERSION} && \ - wget http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip -O /tmp/chromedriver_linux64.zip && \ - unzip /tmp/chromedriver_linux64.zip -d /opt/chromedriver-$CHROMEDRIVER_VERSION && \ - rm /tmp/chromedriver_linux64.zip && \ - chmod +x /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver && \ - ln -fs /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver /usr/local/bin/chromedriver - -# Install Google Chrome -RUN wget http://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${CHROME_VERSION}_amd64.deb -O /tmp/chrome.deb \ - && apt-get -y install /tmp/chrome.deb \ - # Make sure libzstd1 is latest (ID'd as potential vulnerability by Snyk) \ - && apt-get upgrade -y libzstd1 \ - && rm /tmp/chrome.deb - + && apt-get install -y --no-install-recommends libnss3 libdbus-glib-1-2 + +# Install GeckoDriver WebDriver +RUN wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O /tmp/geckodriver.tar.gz && \ + tar xvfz /tmp/geckodriver.tar.gz -C /tmp && \ + mv /tmp/geckodriver /usr/local/bin/geckodriver && \ + rm /tmp/geckodriver.tar.gz + +# Install Firefox +RUN wget https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O /opt/firefox.tar.bz2 && \ + tar xvf /opt/firefox.tar.bz2 -C /opt && \ + ln -s /opt/firefox/firefox /usr/local/bin/firefox # Cache everything for dev purposes... RUN cd /app \ diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index eefde59b27b72..d31cd26df24a9 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -91,20 +91,12 @@ class CeleryConfig(object): WEBDRIVER_BASEURL = "http://superset:8088/" # The base URL for the email report hyperlinks. WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL -WEBDRIVER_TYPE = "chrome" -WEBDRIVER_OPTION_ARGS = [ - "--headless", - "--no-sandbox", - "--disable-dev-shm-usage", - "--disable-crash-reporter", -] -ALERT_REPORTS_NOTIFICATION_DRY_RUN = True - CELERY_CONFIG = CeleryConfig SQLLAB_CTAS_NO_LIMIT = True FEATURE_FLAGS = {"ALERT_REPORTS": True} +PUBLIC_ROLE_LIKE = "Admin" # # Optionally import superset_config_docker.py (which will have been included on # the PYTHONPATH) in order to allow for local settings to be overridden From 6ff9e0b561a02cb0f66d2828bc7cde8232f7e26f Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Thu, 22 Apr 2021 15:10:52 +0100 Subject: [PATCH 4/6] fix config --- docker/pythonpath_dev/superset_config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index d31cd26df24a9..931aacad57044 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -88,15 +88,15 @@ class CeleryConfig(object): } +CELERY_CONFIG = CeleryConfig + +FEATURE_FLAGS = {"ALERT_REPORTS": True} WEBDRIVER_BASEURL = "http://superset:8088/" # The base URL for the email report hyperlinks. WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL -CELERY_CONFIG = CeleryConfig -SQLLAB_CTAS_NO_LIMIT = True -FEATURE_FLAGS = {"ALERT_REPORTS": True} +SQLLAB_CTAS_NO_LIMIT = True -PUBLIC_ROLE_LIKE = "Admin" # # Optionally import superset_config_docker.py (which will have been included on # the PYTHONPATH) in order to allow for local settings to be overridden From 7ca84043b3567ae03702d4599f5ba42d75681b43 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Thu, 22 Apr 2021 15:11:59 +0100 Subject: [PATCH 5/6] fix config --- docker/pythonpath_dev/superset_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index 931aacad57044..a3ca8b1ec6c92 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -91,6 +91,7 @@ class CeleryConfig(object): CELERY_CONFIG = CeleryConfig FEATURE_FLAGS = {"ALERT_REPORTS": True} +ALERT_REPORTS_NOTIFICATION_DRY_RUN = True WEBDRIVER_BASEURL = "http://superset:8088/" # The base URL for the email report hyperlinks. WEBDRIVER_BASEURL_USER_FRIENDLY = WEBDRIVER_BASEURL From e46772363ba2274bc40905c67f9e6aa922431f8f Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Thu, 22 Apr 2021 15:46:15 +0100 Subject: [PATCH 6/6] add missing package --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9bee98a90c334..9d6a61bef9b50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -131,7 +131,7 @@ COPY ./requirements/*.txt ./docker/requirements-*.txt/ /app/requirements/ USER root RUN apt-get update -y \ - && apt-get install -y --no-install-recommends libnss3 libdbus-glib-1-2 + && apt-get install -y --no-install-recommends libnss3 libdbus-glib-1-2 libgtk-3-0 libx11-xcb1 # Install GeckoDriver WebDriver RUN wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O /tmp/geckodriver.tar.gz && \