Adds initial scripts to use rsyslog for logging in workstation

MANIFEST.in

@@ -3,8 +3,9 @@ include README.md
 include changelog.md
 include build-requirements.txt
 include requirements.txt
-include securedrop_log/*.py
-include securedrop_log/VERSION
-include setup.py
-include securedrop-log
+include securedrop-log*
+include securedrop-redis-log
 include securedrop.Log
+include sd-rsyslog*
+include sdlog.conf
+include VERSION

Makefile

@@ -0,0 +1,40 @@
+DEFAULT_GOAL: help
+SHELL := /bin/bash
+
+# Bandit is a static code analysis tool to detect security vulnerabilities in Python applications
+# https://wiki.openstack.org/wiki/Security/Projects/Bandit
+.PHONY: bandit
+bandit: ## Run bandit with medium level excluding test-related folders
+	pip install --upgrade pip && \
+        pip install --upgrade bandit!=1.6.0 && \
+	bandit -ll --recursive . --exclude tests,.venv
+
+.PHONY: safety
+safety: ## Runs `safety check` to check python dependencies for vulnerabilities
+	pip install --upgrade safety && \
+		for req_file in `find . -type f -name '*requirements.txt'`; do \
+			echo "Checking file $$req_file" \
+			&& safety check --full-report -r $$req_file \
+			&& echo -e '\n' \
+			|| exit 1; \
+		done
+
+.PHONY: update-pip-requirements
+update-pip-requirements: ## Updates all Python requirements files via pip-compile.
+	pip-compile --generate-hashes --output-file requirements.txt requirements.in
+
+
+# Explaination of the below shell command should it ever break.
+# 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
+# 2. Use sed-like syntax to remove the make targets
+# 3. Format the split fields into $$1) the target name (in blue) and $$2) the target descrption
+# 4. Pass this file as an arg to awk
+# 5. Sort it alphabetically
+# 6. Format columns with colon as delimiter.
+.PHONY: help
+help: ## Print this message and exit.
+	@printf "Makefile for developing and testing the SecureDrop Logging system.\n"
+	@printf "Subcommands:\n\n"
+	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
+		| sort \
+		| column -s ':' -t

README.md

@@ -24,19 +24,47 @@ Add the following content to `/etc/qubes-rpc/securedrop.Log`
 /usr/sbin/securedrop-log
 ```
 
-and then place `securedrop-log` script to `/usr/sbin/` directory and make sure that
-it is executable.
+and then place `securedrop-redis-log` and `securedrop-log-saver` scripts to the
+virtualenv at `/opt/venvs/securedrop-log` and create links to `/usr/sbin/`
+directory and make sure that they are executable. This step will be automated via
+the Debian package.
+
+
+Copy `securedrop-log.service` file to `/usr/systemd/system` and then
+
+```
+sudo systemctl daemon-reload
+sudo systemctl start redis
+sudo systemctl start securedrop-log
+```
+
+To test the logging, make sure to execute `securedrop-log-saver` from a terminal in `sd-log`
+and check the ~/QubesIncomingLogs/vmname/syslog.log file via **tail -f**.
+
 
 ### To use from any Python code in workvm
 
+Put `sd-rsyslog-example.conf` file to `/etc/sd-rsyslog.conf`, make sure update
+it so that is shows the right **localvm** name.
+
+Copy `sd-rsyslog` executable to **/usr/sbin**, and remember to `chmod +x`
+the binary. 
+
+Next, restart the rsyslog service.
+
+```
+systemctl restart rsyslog
+```
+
+
 Here is an example code using Python logging
 
 ```Python
 import logging
-from securedrop_log import SecureDropLog
+import logging.handlers
 
 def main():
-    handler = SecureDropLog("workvm", "proxy-debian")
+    handler = logging.handlers.SysLogHandler(address="/dev/log")
     logging.basicConfig(level=logging.DEBUG, handlers=[handler])
     logger = logging.getLogger("example")
 
@@ -48,8 +76,9 @@ if __name__ == "__main__":
 
 ```
 
-## The journalctl example
+Or use the logger command.
 
-You will need `python3-systemd` package for the same.
+```
+logger This line should show in the syslog.log file in the sd-log file.
+```
 
-The code is in `journal-example.py` file.

VERSION

@@ -0,0 +1 @@
+0.0.4

build-requirements.txt

@@ -0,0 +1 @@
+redis==3.3.11 --hash=sha256:022f124431ae16ee3a3a69c8016e3e2b057b4f4e0bfa7787b6271d893890c3cc

changelog.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.0.4
+
+  * Converts into rsyslog based logging system.
+
 ## 0.0.3
 
   * Fixes typos MANIFEST.in and setup.py

requirements.in

@@ -0,0 +1 @@
+redis==3.3.11

requirements.txt

@@ -0,0 +1,9 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=requirements.txt requirements.in
+#
+redis==3.3.11 \
+    --hash=sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62 \
+    --hash=sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2

sd-rsyslog

@@ -0,0 +1,193 @@
+#!/opt/venvs/securedrop-log/bin/python3
+"""A skeleton for a Python rsyslog output plugin with error handling.
+Requires Python 3.
+
+To integrate a plugin based on this skeleton with rsyslog, configure an
+'omprog' action like the following:
+    action(type="omprog"
+        binary="/usr/bin/myplugin.py"
+        output="/var/log/myplugin.log"
+        confirmMessages="on"
+        ...)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+     -or-
+     see COPYING.ASL20 in the source distribution
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import sys
+import os
+import logging
+import configparser
+from subprocess import Popen, PIPE
+
+# Global definitions specific to your plugin
+process = None
+
+class RecoverableError(Exception):
+    """An error that has caused the processing of the current message to
+    fail, but does not require restarting the plugin.
+
+    An example of such an error would be a temporary loss of connection to
+    a database or a server. If such an error occurs in the onMessage function,
+    your plugin should wrap it in a RecoverableError before raising it.
+    For example:
+
+        try:
+            # code that connects to a database
+        except DbConnectionError as e:
+            raise RecoverableError from e
+
+    Recoverable errors will cause the 'omprog' action to be temporarily
+    suspended by rsyslog, during a period that can be configured using the
+    "action.resumeInterval" action parameter. When the action is resumed,
+    rsyslog will resend the failed message to your plugin.
+    """
+
+
+def onInit():
+    """Do everything that is needed to initialize processing (e.g. open files,
+    create handles, connect to systems...).
+    """
+    # Apart from processing the logs received from rsyslog, you want your plugin
+    # to be able to report its own logs in some way. This will facilitate
+    # diagnosing problems and debugging your code. Here we set up the standard
+    # Python logging system to output the logs to stderr. In the rsyslog
+    # configuration, you can configure the 'omprog' action to capture the stderr
+    # of your plugin by specifying the action's "output" parameter.
+    logging.basicConfig(stream=sys.stderr,
+                        level=logging.WARNING,
+                        format='%(asctime)s %(levelname)s %(message)s')
+
+    # This is an example of a debug log. (Note that for debug logs to be
+    # emitted you must set 'level' to logging.DEBUG above.)
+    logging.debug("onInit called")
+
+
+    global process
+    if not os.path.exists("/etc/sd-rsyslog.conf"):
+        print("Please create the configuration file at /etc/sd-rsyslog.conf", file=sys.stderr)
+        sys.exit(1)
+    config = configparser.ConfigParser()
+    config.read('/etc/sd-rsyslog.conf')
+    logvmname = config['sd-rsyslog']['remotevm']
+    localvmname = config['sd-rsyslog']['localvm']
+    process = Popen(
+            ["/usr/lib/qubes/qrexec-client-vm", logvmname, "securedrop.Log"],
+            stdin=PIPE,
+            stdout=PIPE,
+            stderr=PIPE,
+    )
+    process.stdin.write(localvmname.encode("utf-8"))
+    process.stdin.write(b"\n")
+    process.stdin.flush()
+
+
+def onMessage(msg):
+    """Process one log message received from rsyslog (e.g. send it to a
+    database). If this function raises an error, the message will be retried
+    by rsyslog.
+
+    Args:
+        msg (str): the log message. Does NOT include a trailing newline.
+
+    Raises:
+        RecoverableError: If a recoverable error occurs. The message will be
+            retried without restarting the plugin.
+        Exception: If a non-recoverable error occurs. The plugin will be
+            restarted before retrying the message.
+    """
+    logging.debug("onMessage called")
+
+    # For illustrative purposes, this plugin skeleton appends the received logs
+    # to a file. When implementing your plugin, remove the following code.
+    global process
+    process.stdin.write(msg.encode("utf-8"))
+    process.stdin.write(b"\n")
+    process.stdin.flush()
+
+
+def onExit():
+    """Do everything that is needed to finish processing (e.g. close files,
+    handles, disconnect from systems...). This is being called immediately
+    before exiting.
+
+    This function should not raise any error. If it does, the error will be
+    logged as a warning and ignored.
+    """
+    logging.debug("onExit called")
+
+    # For illustrative purposes, this plugin skeleton appends the received logs
+    # to a file. When implementing your plugin, remove the following code.
+    global process
+    process.stdin.flush()
+
+
+"""
+-------------------------------------------------------
+This is plumbing that DOES NOT need to be CHANGED
+-------------------------------------------------------
+This is the main loop that receives messages from rsyslog via stdin,
+invokes the above entrypoints, and provides status codes to rsyslog
+via stdout. In most cases, modifying this code should not be necessary.
+"""
+try:
+    onInit()
+except Exception as e:
+    # If an error occurs during initialization, log it and terminate. The
+    # 'omprog' action will eventually restart the program.
+    logging.exception("Initialization error, exiting program")
+    sys.exit(1)
+
+# Tell rsyslog we are ready to start processing messages:
+print("OK", flush=True)
+
+endedWithError = False
+try:
+    line = sys.stdin.readline()
+    while line:
+        line = line.rstrip('\n')
+        try:
+            onMessage(line)
+            status = "OK"
+        except RecoverableError as e:
+            # Any line written to stdout that is not a status code will be
+            # treated as a recoverable error by 'omprog', and cause the action
+            # to be temporarily suspended. In this skeleton, we simply return
+            # a one-line representation of the Python exception. (If debugging
+            # is enabled in rsyslog, this line will appear in the debug logs.)
+            status = repr(e)
+            # We also log the complete exception to stderr (or to the logging
+            # handler(s) configured in doInit, if any).
+            logging.exception(e)
+
+        # Send the status code (or the one-line error message) to rsyslog:
+        print(status, flush=True)
+        line = sys.stdin.readline()
+
+except Exception:
+    # If a non-recoverable error occurs, log it and terminate. The 'omprog'
+    # action will eventually restart the program.
+    logging.exception("Unrecoverable error, exiting program")
+    endedWithError = True
+
+try:
+    onExit()
+except Exception:
+    logging.warning("Exception ignored in onExit", exc_info=True)
+
+if endedWithError:
+    sys.exit(1)
+else:
+    sys.exit(0)
+

sd-rsyslog-example.conf

@@ -0,0 +1,4 @@
+[sd-rsyslog]
+remotevm = sd-log
+localvm = sd-app
+

sdlog.conf

@@ -0,0 +1,4 @@
+module(load="omprog")
+action(type="omprog"
+       binary="/usr/sbin/sd-rsyslog"
+       template="RSYSLOG_TraditionalFileFormat")