Update postfix headers

apps/common/src/python/mediawords/util/mail.py

@@ -117,6 +117,9 @@ def send_email(message: Message) -> bool:
             message_part = MIMEText(message.text_body, 'plain', 'utf-8')
             mime_message.attach(message_part)
 
+        mime_message.add_header('List-Unsubscribe',
+                                'mailto:[email protected]?subject=Delete%20account%20and%20unsubscribe')
+
         # HTML gets attached last, thus making it a preferred part as per RFC
         if message.html_body:
             message_part = MIMEText(message.html_body, 'html', 'utf-8')

apps/mail-postfix-server/Dockerfile

@@ -50,13 +50,16 @@ RUN \
     #
     # Filter out "Received:" and some other headers
     postconf -e header_checks=regexp:/etc/postfix/header_checks && \
-    postconf -e mime_header_checks=regexp:/etc/postfix/header_checks && \
     postconf -e smtp_header_checks=regexp:/etc/postfix/header_checks && \
     #
     # Don't require TLS as local clients are trusted
     postconf -e smtp_tls_security_level=may && \
     postconf -e smtpd_tls_security_level=none && \
     #
+    # Make sure default headers (e.g. Message-Id, date) are present
+    postconf -e always_add_missing_headers=yes && \
+    postconf -e local_header_rewrite_clients=permit_inet_interfaces && \
+    #
     # Disable chroot on all services as it doesn't play well with a mounted
     # volume, e.g. "smtpd" is unable to access libnss after a chroot and thus
     # resolve OpenDKIM container.

apps/mail-postfix-server/docker-compose.tests.yml

@@ -4,15 +4,36 @@ services:
 
     # Service to use for testing the mail service
     #
-    # Usage:
+    # Steps to test:
     #
-    #     host$ ./dev/run.py mail-postfix-server bash
-    #     container$ sendmail "[email protected]"
+    #     1) host$ ./dev/run.py mail-postfix-server bash
+    #     2) (new terminal window) host$ docker ps
+    #     3) find container with name ending in 'mail-postfix-server-actual_1'
+    #     4) host$ docker exec -it some_string_mail-postfix-server-actual_1 bash
+    #     5) container$ ./postfix.sh
+    #     6) open new terminal window on your host machine
+    #     7) host$ docker exec -it some_string_mail-postfix-server-actual_1 bash
+    #     8) follow instructions at URL below to create a test mail.txt file and send to your email address from the container
+    #     https://clients.javapipe.com/knowledgebase/132/How-to-Test-Sendmail-From-Command-Line-on-Linux.html
     #
+    #     Alternatively, if you want to test via the send_email() method (https://github.com/mediacloud/backend/blob/master/apps/common/src/python/mediawords/util/mail.py#L73),
+    #     or test changes to said method, to you can disregard steps 7-8 above and instead:
+    #     7) host$ docker ps
+    #     8) Find mail-postfix-server container ID
+    #     9) host$ docker exec -it some_string_mail-postfix-server
+    #     10) $container python3
+    #     11) >> from mediawords.util.mail import *
+    #     12) >> test_message = Message(to='[email protected]', subject='test postfix', text_body=None, html_body='<p>hi</p>')
+    #     13) >> send_email(test_message)
+    #    
     mail-postfix-server:
         image: gcr.io/mcback/common:latest
         init: true
         stop_signal: SIGKILL
+        volumes:
+            - type: bind
+              source: ./../common/src/
+              target: /opt/mediacloud/src/common/
         depends_on:
             - mail-postfix-server-actual
 
@@ -21,7 +42,7 @@ services:
         image: gcr.io/mcback/mail-postfix-server:latest
         init: true
         stop_signal: SIGKILL
-        # "docker exec" into a container and run Postfix manually (/postfix.sh):
+        # "docker exec" into a container and run Postfix manually (./postfix.sh):
         command: sleep infinity
         # To be able to set /proc/sys/kernel/yama/ptrace_scope:
         privileged: true

apps/mail-postfix-server/header_checks

@@ -1,5 +1,4 @@
 /^Received:.*with ESMTP /         IGNORE
 /^X-Originating-IP:/    IGNORE
 /^X-Mailer:/            IGNORE
-/^Mime-Version:/        IGNORE
 /^User-Agent:/          IGNORE

apps/topics-map/Dockerfile

@@ -5,7 +5,10 @@
 FROM gcr.io/mcback/common:latest
 
 # Install Java
-RUN apt-get -y --no-install-recommends install openjdk-8-jre-headless
+RUN \ 
+    apt-get update && \ 
+    apt-get -y --no-install-recommends install openjdk-8-jre-headless && \
+    true
 
 # Install fa2l Java libs
 RUN \