Update postfix headers

apps/common/docker-compose.tests.yml

@@ -13,6 +13,12 @@ services:
             MC_DOWNLOADS_AMAZON_S3_DIRECTORY_NAME: "${MC_DOWNLOADS_AMAZON_S3_DIRECTORY_NAME}"
             MC_PUBLIC_STORE_TYPE: "postgresql"
             MC_PUBLIC_STORE_SALT: "foo"
+            # Email address to point to in List-Unsubscribe email header.
+            # Technically we don't have a straightforward "unsubscribe" endpoint, but our 
+            # emails are more likely to be marked spam if we don't have such a header, so
+            # we make the email subject "Delete account and unsubscribe" in 
+            # mediawords/util/config/common.py
+            MC_EMAIL_UNSUBSCRIBE: "[email protected]"
         volumes:
             - type: bind
               source: ./src/

apps/common/src/python/mediawords/util/config/common.py

@@ -155,6 +155,15 @@ def password() -> str:
         """Password."""
         return ''
 
+    @staticmethod
+    def unsubscribe_address() -> str:
+        """Mailto link for email to which unsubscribe/account deletion requests should be sent"""
+        address = env_value('MC_EMAIL_UNSUBSCRIBE', required=False, allow_empty_string=True)
+        if address is None or '@' not in address:
+            address = '[email protected]'
+        return address
+        # return f'mailto:{address}?subject=Delete%20account%20and%20unsubscribe'
+
 
 class DownloadStorageConfig(object):
     """Download storage configuration."""

apps/common/src/python/mediawords/util/mail.py

@@ -117,6 +117,12 @@ def send_email(message: Message) -> bool:
             message_part = MIMEText(message.text_body, 'plain', 'utf-8')
             mime_message.attach(message_part)
 
+        unsubscribe_address = CommonConfig.smtp().unsubscribe_address()
+
+        mime_message.add_header(
+            'List-Unsubscribe', 
+             f'mailto:{unsubscribe_address}?subject=Delete%20account%20and%20unsubscribe')
+
         # HTML gets attached last, thus making it a preferred part as per RFC
         if message.html_body:
             message_part = MIMEText(message.html_body, 'html', 'utf-8')

apps/docker-compose.dist.yml

@@ -74,6 +74,14 @@ x-common-configuration: &common-configuration
     # "From:" email address when sending emails
     MC_EMAIL_FROM_ADDRESS: "[email protected]"
 
+    # Email address to point to in List-Unsubscribe email header.
+    # Technically we don't have a straightforward "unsubscribe" endpoint, but our 
+    # emails are more likely to be marked spam if we don't have such a header, so
+    # we make the email subject "Delete account and unsubscribe" in 
+    # mediawords/util/config/common.py
+    # example value = [email protected] 
+    MC_EMAIL_UNSUBSCRIBE: "[email protected]"
+
     # Fail all HTTP requests that match the following pattern, e.g.
     # "^https?://[^/]*some-website.com"
     MC_USERAGENT_BLACKLIST_URL_PATTERN: ""

apps/mail-postfix-server/Dockerfile

@@ -50,13 +50,16 @@ RUN \
     #
     # Filter out "Received:" and some other headers
     postconf -e header_checks=regexp:/etc/postfix/header_checks && \
-    postconf -e mime_header_checks=regexp:/etc/postfix/header_checks && \
     postconf -e smtp_header_checks=regexp:/etc/postfix/header_checks && \
     #
     # Don't require TLS as local clients are trusted
     postconf -e smtp_tls_security_level=may && \
     postconf -e smtpd_tls_security_level=none && \
     #
+    # Make sure default headers (e.g. Message-Id, date) are present
+    postconf -e always_add_missing_headers=yes && \
+    postconf -e local_header_rewrite_clients=permit_inet_interfaces && \
+    #
     # Disable chroot on all services as it doesn't play well with a mounted
     # volume, e.g. "smtpd" is unable to access libnss after a chroot and thus
     # resolve OpenDKIM container.

apps/mail-postfix-server/docker-compose.tests.yml

@@ -4,15 +4,36 @@ services:
 
     # Service to use for testing the mail service
     #
-    # Usage:
+    # Steps to test:
     #
-    #     host$ ./dev/run.py mail-postfix-server bash
-    #     container$ sendmail "[email protected]"
+    #     1) host$ ./dev/run.py mail-postfix-server bash
+    #     2) (new terminal window) host$ docker ps
+    #     3) find container with name ending in 'mail-postfix-server-actual_1'
+    #     4) host$ docker exec -it some_string_mail-postfix-server-actual_1 bash
+    #     5) container$ ./postfix.sh
+    #     6) open new terminal window on your host machine
+    #     7) host$ docker exec -it some_string_mail-postfix-server-actual_1 bash
+    #     8) follow instructions at URL below to create a test mail.txt file and send to your email address from the container
+    #     https://clients.javapipe.com/knowledgebase/132/How-to-Test-Sendmail-From-Command-Line-on-Linux.html
     #
+    #     Alternatively, if you want to test via the send_email() method (https://github.com/mediacloud/backend/blob/master/apps/common/src/python/mediawords/util/mail.py#L73),
+    #     or test changes to said method, to you can disregard steps 7-8 above and instead:
+    #     7) host$ docker ps
+    #     8) Find mail-postfix-server container ID
+    #     9) host$ docker exec -it some_string_mail-postfix-server
+    #     10) $container python3
+    #     11) >> from mediawords.util.mail import *
+    #     12) >> test_message = Message(to='[email protected]', subject='test postfix', text_body=None, html_body='<p>hi</p>')
+    #     13) >> send_email(test_message)
+    #    
     mail-postfix-server:
         image: gcr.io/mcback/common:latest
         init: true
         stop_signal: SIGKILL
+        volumes:
+            - type: bind
+              source: ./../common/src/
+              target: /opt/mediacloud/src/common/
         depends_on:
             - mail-postfix-server-actual
 
@@ -21,7 +42,7 @@ services:
         image: gcr.io/mcback/mail-postfix-server:latest
         init: true
         stop_signal: SIGKILL
-        # "docker exec" into a container and run Postfix manually (/postfix.sh):
+        # "docker exec" into a container and run Postfix manually (./postfix.sh):
         command: sleep infinity
         # To be able to set /proc/sys/kernel/yama/ptrace_scope:
         privileged: true

apps/mail-postfix-server/header_checks

@@ -1,5 +1,4 @@
 /^Received:.*with ESMTP /         IGNORE
 /^X-Originating-IP:/    IGNORE
 /^X-Mailer:/            IGNORE
-/^Mime-Version:/        IGNORE
 /^User-Agent:/          IGNORE

apps/topics-map/Dockerfile

@@ -5,7 +5,10 @@
 FROM gcr.io/mcback/common:latest
 
 # Install Java
-RUN apt-get -y --no-install-recommends install openjdk-8-jre-headless
+RUN \ 
+    apt-get -y update && \ 
+    apt-get -y --no-install-recommends install openjdk-8-jre-headless && \
+    true
 
 # Install fa2l Java libs
 RUN \