work on issue #102, log access to Malcolm web interface(s) to Elastic…

…search for analysis in Kibana

Dockerfiles/filebeat.Dockerfile

@@ -10,7 +10,6 @@ LABEL org.opencontainers.image.vendor='Idaho National Laboratory'
 LABEL org.opencontainers.image.title='malcolmnetsec/filebeat-oss'
 LABEL org.opencontainers.image.description='Malcolm container providing Filebeat (the Apache-licensed variant)'
 
-
 ARG FILEBEAT_LOG_CLEANUP_MINUTES=0
 ARG FILEBEAT_ZIP_CLEANUP_MINUTES=0
 ARG FILEBEAT_SCAN_FREQUENCY=10s
@@ -22,6 +21,8 @@ ARG FILEBEAT_CLOSE_REMOVED=true
 ARG FILEBEAT_CLOSE_EOF=true
 ARG FILEBEAT_CLEAN_REMOVED=true
 ARG FILEBEAT_LOG_PATH="/data/zeek/current"
+ARG FILEBEAT_NGINX_LOG_PATH="/data/nginx"
+ARG NGINX_LOG_ACCESS_AND_ERRORS=false
 ARG AUTO_TAG=true
 
 USER root
@@ -36,10 +37,16 @@ RUN yum install -y epel-release && \
 
 ADD shared/bin/cron_env_centos.sh /data/
 ADD filebeat/filebeat.yml /usr/share/filebeat/filebeat.yml
+ADD filebeat/filebeat-nginx.yml /usr/share/filebeat-nginx/filebeat-nginx.yml
 ADD filebeat/scripts /data/
+ADD shared/bin/elastic_search_status.sh /data/
 ADD filebeat/supervisord.conf /etc/supervisord.conf
-RUN chmod 755 /data/*.sh /data/*.py && \
-    mkdir -p /var/log/supervisor && \
+RUN mkdir -p /var/log/supervisor /usr/share/filebeat-nginx/data && \
+    chown -R root:filebeat /usr/share/filebeat-nginx && \
+    cp -a /usr/share/filebeat/module /usr/share/filebeat-nginx/module && \
+    chmod 750 /usr/share/filebeat-nginx && \
+    chmod 770 /usr/share/filebeat-nginx/data && \
+    chmod 755 /data/*.sh /data/*.py && \
     (echo -e "* * * * * su -c /data/filebeat-process-zeek-folder.sh filebeat >/dev/null 2>&1\n*/5 * * * * su -c /data/filebeat-clean-zeeklogs-processed-folder.py filebeat >/dev/null 2>&1" | crontab -)
 
 ENV FILEBEAT_LOG_CLEANUP_MINUTES $FILEBEAT_LOG_CLEANUP_MINUTES
@@ -53,13 +60,15 @@ ENV FILEBEAT_CLOSE_REMOVED $FILEBEAT_CLOSE_REMOVED
 ENV FILEBEAT_CLOSE_EOF $FILEBEAT_CLOSE_EOF
 ENV FILEBEAT_CLEAN_REMOVED $FILEBEAT_CLEAN_REMOVED
 ENV FILEBEAT_LOG_PATH $FILEBEAT_LOG_PATH
+ENV FILEBEAT_NGINX_LOG_PATH $FILEBEAT_NGINX_LOG_PATH
+ENV NGINX_LOG_ACCESS_AND_ERRORS $NGINX_LOG_ACCESS_AND_ERRORS
 ENV AUTO_TAG $AUTO_TAG
 
 ENV FILEBEAT_REGISTRY_FILE "/usr/share/filebeat/data/registry/filebeat/data.json"
 ENV FILEBEAT_ZEEK_DIR "/data/zeek/"
 ENV PATH="/data:${PATH}"
 
-VOLUME ["/usr/share/filebeat/data"]
+VOLUME ["/usr/share/filebeat/data", "/usr/share/filebeat-nginx/data"]
 
 CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-u", "root", "-n"]
 

Dockerfiles/nginx.Dockerfile

@@ -207,8 +207,6 @@ RUN set -x ; \
   mv /tmp/envsubst /usr/local/bin/ ; \
   mkdir -p /var/log/supervisor ; \
   rm -rf /usr/src/* /var/tmp/* /var/cache/apk/* /tmp/stunnel-*.apk /nginx.tar.gz /nginx-auth-ldap.tar.gz /ngx_http_auth_pam_module.tar.gz /docker-gen-alpine-linux-amd64-$DOCKER_GEN_VERSION.tar.gz; \
-  ln -sf /dev/stdout /var/log/nginx/access.log; \
-  ln -sf /dev/stderr /var/log/nginx/error.log; \
   touch /etc/nginx/nginx_ldap.conf /etc/nginx/nginx_blank.conf;
 
 COPY --from=jwilder/nginx-proxy:alpine /app/nginx.tmpl /etc/nginx/

README.md

@@ -424,6 +424,8 @@ Various other environment variables inside of `docker-compose.yml` can be tweake
 
 * `NGINX_BASIC_AUTH` - if set to `true`, use [TLS-encrypted HTTP basic](#AuthBasicAccountManagement) authentication (default); if set to `false`, use [Lightweight Directory Access Protocol (LDAP)](#AuthLDAP) authentication
 
+* `NGINX_LOG_ACCESS_AND_ERRORS` - if set to `true`, all access to Malcolm via its [web interfaces](#UserInterfaceURLs) will be logged to Elasticsearch (default `false`)
+
 * `MANAGE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will be marked as available for deletion by Moloch if available storage space becomes too low (default `false`)
 
 * `ZEEK_AUTO_ANALYZE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will automatically be analyzed by Zeek, and the resulting logs will also be imported (default `false`)

docker-compose-standalone-zeek-live.yml

@@ -16,6 +16,10 @@ x-auth-variables: &auth-variables
   # "extendedReq(1) LDAP_START_TLS_OID" command: winldap or openldap
   NGINX_LDAP_TLS_STUNNEL_PROTOCOL : 'winldap'
 
+x-nginx-variables: &nginx-variables
+  # Whether or not to write nginx's access.log and error.log to Elasticsearch
+  NGINX_LOG_ACCESS_AND_ERRORS : 'false'
+
 x-moloch-variables: &moloch-variables
   MANAGE_PCAP_FILES : 'false'
   MOLOCH_ANALYZE_PCAP_THREADS : 1
@@ -212,9 +216,11 @@ services:
     restart: "no"
     hostname: filebeat
     environment:
+      << : *nginx-variables
       << : *common-upload-variables
       << : *common-beats-variables
       FILEBEAT_LOG_PATH : '/data/zeek/current'
+      FILEBEAT_NGINX_LOG_PATH : '/data/nginx'
       FILEBEAT_LOG_CLEANUP_MINUTES : 180
       FILEBEAT_ZIP_CLEANUP_MINUTES : 360
       FILEBEAT_SCAN_FREQUENCY : '10s'
@@ -228,6 +234,7 @@ services:
     depends_on:
       - logstash
     volumes:
+      - nginx-log-path:/data/nginx:ro
       - ./zeek-logs:/data/zeek
       - ./filebeat/certs/ca.crt:/certs/ca.crt:ro
       - ./filebeat/certs/client.crt:/certs/client.crt:ro
@@ -363,6 +370,7 @@ services:
     hostname: nginx-proxy
     environment:
       << : *auth-variables
+      << : *nginx-variables
     depends_on:
       - moloch
       - kibana
@@ -378,9 +386,13 @@ services:
       - "9600:9600"
       - "28991:28991"
     volumes:
+      - nginx-log-path:/var/log/nginx:rw
       - /var/run/docker.sock:/tmp/docker.sock:ro
       - ./nginx/nginx_ldap.conf:/etc/nginx/nginx_ldap.conf:ro
       - ./nginx/htpasswd:/etc/nginx/.htpasswd:ro
       - ./nginx/certs:/etc/nginx/certs:ro
       - ./nginx/certs/dhparam.pem:/etc/nginx/dhparam/dhparam.pem:ro
 
+# shared named volume so filebeat can access nginx access logs
+volumes:
+  nginx-log-path:

docker-compose-standalone.yml

@@ -16,6 +16,10 @@ x-auth-variables: &auth-variables
   # "extendedReq(1) LDAP_START_TLS_OID" command: winldap or openldap
   NGINX_LDAP_TLS_STUNNEL_PROTOCOL : 'winldap'
 
+x-nginx-variables: &nginx-variables
+  # Whether or not to write nginx's access.log and error.log to Elasticsearch
+  NGINX_LOG_ACCESS_AND_ERRORS : 'false'
+
 x-moloch-variables: &moloch-variables
   MANAGE_PCAP_FILES : 'false'
   MOLOCH_ANALYZE_PCAP_THREADS : 1
@@ -212,9 +216,11 @@ services:
     restart: "no"
     hostname: filebeat
     environment:
+      << : *nginx-variables
       << : *common-upload-variables
       << : *common-beats-variables
       FILEBEAT_LOG_PATH : '/data/zeek/current'
+      FILEBEAT_NGINX_LOG_PATH : '/data/nginx'
       FILEBEAT_LOG_CLEANUP_MINUTES : 180
       FILEBEAT_ZIP_CLEANUP_MINUTES : 360
       FILEBEAT_SCAN_FREQUENCY : '10s'
@@ -228,6 +234,7 @@ services:
     depends_on:
       - logstash
     volumes:
+      - nginx-log-path:/data/nginx:ro
       - ./zeek-logs:/data/zeek
       - ./filebeat/certs/ca.crt:/certs/ca.crt:ro
       - ./filebeat/certs/client.crt:/certs/client.crt:ro
@@ -363,6 +370,7 @@ services:
     hostname: nginx-proxy
     environment:
       << : *auth-variables
+      << : *nginx-variables
     depends_on:
       - moloch
       - kibana
@@ -378,9 +386,13 @@ services:
       - "9600:9600"
       - "28991:28991"
     volumes:
+      - nginx-log-path:/var/log/nginx:rw
       - /var/run/docker.sock:/tmp/docker.sock:ro
       - ./nginx/nginx_ldap.conf:/etc/nginx/nginx_ldap.conf:ro
       - ./nginx/htpasswd:/etc/nginx/.htpasswd:ro
       - ./nginx/certs:/etc/nginx/certs:ro
       - ./nginx/certs/dhparam.pem:/etc/nginx/dhparam/dhparam.pem:ro
 
+# shared named volume so filebeat can access nginx access logs
+volumes:
+  nginx-log-path:

docker-compose.yml

@@ -16,6 +16,10 @@ x-auth-variables: &auth-variables
   # "extendedReq(1) LDAP_START_TLS_OID" command: winldap or openldap
   NGINX_LDAP_TLS_STUNNEL_PROTOCOL : 'winldap'
 
+x-nginx-variables: &nginx-variables
+  # Whether or not to write nginx's access.log and error.log to Elasticsearch
+  NGINX_LOG_ACCESS_AND_ERRORS : 'false'
+
 x-moloch-variables: &moloch-variables
   MANAGE_PCAP_FILES : 'false'
   MOLOCH_ANALYZE_PCAP_THREADS : 1
@@ -231,9 +235,11 @@ services:
     restart: "no"
     hostname: filebeat
     environment:
+      << : *nginx-variables
       << : *common-upload-variables
       << : *common-beats-variables
       FILEBEAT_LOG_PATH : '/data/zeek/current'
+      FILEBEAT_NGINX_LOG_PATH : '/data/nginx'
       FILEBEAT_LOG_CLEANUP_MINUTES : 180
       FILEBEAT_ZIP_CLEANUP_MINUTES : 360
       FILEBEAT_SCAN_FREQUENCY : '10s'
@@ -247,6 +253,7 @@ services:
     depends_on:
       - logstash
     volumes:
+      - nginx-log-path:/data/nginx:ro
       - ./zeek-logs:/data/zeek
       - ./filebeat/certs/ca.crt:/certs/ca.crt:ro
       - ./filebeat/certs/client.crt:/certs/client.crt:ro
@@ -410,6 +417,7 @@ services:
     hostname: nginx-proxy
     environment:
       << : *auth-variables
+      << : *nginx-variables
     depends_on:
       - moloch
       - kibana
@@ -425,10 +433,14 @@ services:
       - "9600:9600"
       - "28991:28991"
     volumes:
+      - nginx-log-path:/var/log/nginx:rw
       - /var/run/docker.sock:/tmp/docker.sock:ro
       - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
       - ./nginx/nginx_ldap.conf:/etc/nginx/nginx_ldap.conf:ro
       - ./nginx/htpasswd:/etc/nginx/.htpasswd:ro
       - ./nginx/certs:/etc/nginx/certs:ro
       - ./nginx/certs/dhparam.pem:/etc/nginx/dhparam/dhparam.pem:ro
 
+# shared named volume so filebeat can access nginx access logs
+volumes:
+  nginx-log-path:

filebeat/filebeat-nginx.yml

@@ -0,0 +1,40 @@
+# Copyright (c) 2019 Battelle Energy Alliance, LLC.  All rights reserved.
+
+#================================ Modules ======================================
+filebeat.modules:
+- module: nginx
+  access:
+    enabled: true
+    var.paths: ["${FILEBEAT_NGINX_LOG_PATH:/data/nginx}/access.log*"]
+  error:
+    enabled: true
+    var.paths: ["${FILEBEAT_NGINX_LOG_PATH:/data/nginx}/error.log*"]
+
+#================================ Outputs ======================================
+
+#-------------------------- Elasticsearch output -------------------------------
+output.elasticsearch:
+  enabled: true
+  hosts: ["elasticsearch:9200"]
+  indices:
+    - index: "filebeat-%{[agent.version]}-nginx-%{+yyyy.MM.dd}"
+      when.equals:
+        event.module: "nginx"
+
+setup.template.enabled: true
+setup.template.overwrite: false
+setup.template.settings:
+  index.number_of_shards: 1
+  index.number_of_replicas: 0
+
+#============================== Dashboards =====================================
+setup.dashboards.enabled: true
+setup.dashboards.directory: "/usr/share/filebeat/kibana"
+
+#============================== Kibana =====================================
+setup.kibana:
+  host: "kibana:5601"
+  path: "/kibana"
+
+#================================ Logging ======================================
+logging.metrics.enabled: false

filebeat/supervisord.conf

@@ -28,6 +28,24 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 redirect_stderr=true
 
+[program:filebeat-nginx]
+command=bash -c "/data/elastic_search_status.sh && /usr/local/bin/docker-entrypoint -e --strict.perms=false \
+  --path.home /usr/share/filebeat-nginx \
+  --path.config /usr/share/filebeat-nginx \
+  --path.data /usr/share/filebeat-nginx/data \
+  -c /usr/share/filebeat-nginx/filebeat-nginx.yml \
+  --modules nginx"
+user=filebeat
+autostart=%(ENV_NGINX_LOG_ACCESS_AND_ERRORS)s
+startsecs=30
+startretries=2000000000
+stopasgroup=true
+killasgroup=true
+directory=/usr/share/filebeat-nginx
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
 [program:watch-upload]
 command=/bin/bash -c "sleep 30 && /data/filebeat-watch-zeeklogs-uploads-folder.sh"
 startsecs=35

nginx/supervisord.conf

@@ -26,6 +26,30 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 redirect_stderr=true
 
+[program:logaccess]
+command=/usr/bin/tail -F /var/log/nginx/access.log
+startsecs=10
+startretries=2000000000
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/null
+stdout_logfile_maxbytes=0
+redirect_stderr=false
+
+[program:logerrors]
+command=/usr/bin/tail -F /var/log/nginx/error.log
+startsecs=10
+startretries=2000000000
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/null
+stdout_logfile_maxbytes=0
+redirect_stderr=false
+
 [program:dockergen]
 command=/usr/local/bin/docker-gen -watch -notify "nginx -s reload" /etc/nginx/nginx.tmpl /etc/nginx/conf.d/default.conf
 startsecs=5

sensor-iso/moloch/etc/.gitignore

@@ -0,0 +1,4 @@
+GeoLite2-*
+oui.txt*
+ipv4-address-space.csv*
+