diff --git a/CHANGES.rst b/CHANGES.rst index 4e23bd22..7748d15f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,9 @@ in progress - [file] Allow writing of binary content. Thanks, @sevmonster. - [ux] Rename subcommand ``mqttwarn make-samplefuncs`` to ``mqttwarn make-udf``, and adjust naming. +- [ntfy] Add dedicated service plugin ``ntfy`` +- [ntfy] Use RFC 2047 for encoding HTTP header values +- [ntfy] Add more fields: icon, cache, firebase, unifiedpush 2023-04-11 0.33.0 diff --git a/README.rst b/README.rst index 7ae807c2..4b843c64 100644 --- a/README.rst +++ b/README.rst @@ -183,18 +183,22 @@ you an idea how to pass relevant information on the command line using JSON:: # Launch "pushover" service plugin mqttwarn --plugin=pushover --options='{"title": "About", "message": "Hello world", "addrs": ["userkey", "token"], "priority": 6}' - # Launch "ssh" service plugin from the command line + # Launch "ntfy" service plugin + mqttwarn --plugin=ntfy --options='{"addrs": {"url": "http://localhost:5555/testdrive"}, "title": "Example notification", "message": "Hello world"}' --data='{"tags": "foo,bar,äöü", "priority": "high"}' + + # Launch "ntfy" service plugin, and add remote attachment + mqttwarn --plugin=ntfy --options='{"addrs": {"url": "http://localhost:5555/testdrive"}, "title": "Example notification", "message": "Hello world"}' --data='{"attach": "https://unsplash.com/photos/spdQ1dVuIHw/download?w=320", "filename": "goat.jpg"}' + + # Launch "ntfy" service plugin, and add attachment from local filesystem + mqttwarn --plugin=ntfy --options='{"addrs": {"url": "http://localhost:5555/testdrive", "file": "goat.jpg"}, "title": "Example notification", "message": "Hello world"}' + + # Launch "ssh" service plugin mqttwarn --plugin=ssh --config='{"host": "ssh.example.org", "port": 22, "user": "foo", "password": "bar"}' --options='{"addrs": ["command with substitution %s"], "payload": "{\"args\": \"192.168.0.1\"}"}' - # Launch "cloudflare_zone" service plugin from "mqttwarn-contrib", passing "--config" parameters via command line + # Launch "cloudflare_zone" service plugin from "mqttwarn-contrib" pip install mqttwarn-contrib mqttwarn --plugin=mqttwarn_contrib.services.cloudflare_zone --config='{"auth-email": "foo", "auth-key": "bar"}' --options='{"addrs": ["0815", "www.example.org", ""], "message": "192.168.0.1"}' - # Submit notification to "ntfy", using Apprise service plugin. - mqttwarn --plugin=apprise \ - --config='{"baseuri": "ntfy://user:password@ntfy.example.org/topic1/topic2"}' \ - --options='{"addrs": [], "title": "Example notification", "message": "Hello world"}' - Also, the ``--config-file`` parameter can be used to optionally specify the path to a configuration file. diff --git a/docs/notifier-catalog.md b/docs/notifier-catalog.md index 670c4422..218a0bdd 100644 --- a/docs/notifier-catalog.md +++ b/docs/notifier-catalog.md @@ -223,14 +223,14 @@ template arguments. mqttwarn supports propagating them from either the ``baseuri`` configuration setting, or from its data dictionary to the Apprise plugin invocation. -So, for example, you can propagate parameters to the [Apprise Ntfy plugin] -by either pre-setting them as URL query parameters, like +So, for example, you can propagate parameters to the [Apprise JSON HTTP POST +Notifications plugin] by either pre-setting them as URL query parameters, like ``` -ntfy://user:password@ntfy.example.org/topic1/topic2?email=test@example.org +json://localhost/?:sound=oceanwave ``` or by submitting them within a JSON-formatted MQTT message, like ```json -{"priority": "high", "tags": "foo,bar", "click": "https://httpbin.org/headers"} +{":sound": "oceanwave", "tags": "foo,bar", "click": "https://httpbin.org/headers"} ``` @@ -238,7 +238,7 @@ or by submitting them within a JSON-formatted MQTT message, like [Apprise documentation]: https://github.com/caronc/apprise/wiki [Apprise URL Basics]: https://github.com/caronc/apprise/wiki/URLBasics [Apprise Notification Services]: https://github.com/caronc/apprise/wiki#notification-services -[Apprise Ntfy plugin]: https://github.com/caronc/apprise/wiki/Notify_ntfy +[Apprise JSON HTTP POST Notifications plugin]: https://github.com/caronc/apprise/wiki/Notify_Custom_JSON ### `apprise_single` @@ -256,7 +256,7 @@ Apprise to E-Mail, an HTTP endpoint, and a Discord channel. ```ini [defaults] -launch = apprise-mail, apprise-json, apprise-discord, apprise-ntfy +launch = apprise-mail, apprise-json, apprise-discord [config:apprise-mail] ; Dispatch message as e-mail. @@ -283,16 +283,9 @@ baseuri = 'json://localhost:1234/mqtthook' module = 'apprise_single' baseuri = 'discord://4174216298/JHMHI8qBe7bk2ZwO5U711o3dV_js' -[config:apprise-ntfy] -; Dispatch message to ntfy. -; https://github.com/caronc/apprise/wiki/URLBasics -; https://github.com/caronc/apprise/wiki/Notify_ntfy -module = 'apprise_single' -baseuri = 'ntfy://user:password@ntfy.example.org/topic1/topic2' - [apprise-single-test] topic = apprise/single/# -targets = apprise-mail:demo, apprise-json, apprise-discord, apprise-ntfy +targets = apprise-mail:demo, apprise-json, apprise-discord format = Alarm from {device}: {payload} title = Alarm from {device} ``` @@ -325,7 +318,6 @@ module = 'apprise_multi' targets = { 'demo-http' : [ { 'baseuri': 'json://localhost:1234/mqtthook' }, { 'baseuri': 'json://daq.example.org:5555/foobar' } ], 'demo-discord' : [ { 'baseuri': 'discord://4174216298/JHMHI8qBe7bk2ZwO5U711o3dV_js' } ], - 'demo-ntfy' : [ { 'baseuri': 'ntfy://user:password@ntfy.example.org/topic1/topic2' } ], 'demo-mailto' : [ { 'baseuri': 'mailtos://smtp_username:smtp_password@mail.example.org', 'recipients': ['foo@example.org', 'bar@example.org'], @@ -336,7 +328,7 @@ targets = { [apprise-multi-test] topic = apprise/multi/# -targets = apprise-multi:demo-http, apprise-multi:demo-discord, apprise-multi:demo-mailto, apprise-multi:demo-ntfy +targets = apprise-multi:demo-http, apprise-multi:demo-discord, apprise-multi:demo-mailto format = Alarm from {device}: {payload} title = Alarm from {device} ``` @@ -1746,10 +1738,154 @@ Requires: ### `ntfy` -Support for [ntfy] is provided through Apprise, see [apprise_single](#apprise_single) -and [apprise_multi](#apprise_multi). +> [ntfy] (pronounce: _notify_) is a simple HTTP-based [pub-sub] notification service. +> It allows you to send notifications to your phone or desktop via scripts from +> any computer, entirely without signup, cost or setup. +> [ntfy is also open source](https://github.com/binwiederhier/ntfy), if you want to +> run an instance on your own premises. + +ntfy uses topics to address communication channels. This topic is part of the +HTTP API URL. + +To use the hosted variant on `ntfy.sh`, just provide an URL including the topic. +```ini +[config:ntfy] +targets = { + 'test': 'https://ntfy.sh/testdrive', + } +``` + +When running your own instance, you would use a custom URL here. +```ini +[config:ntfy] +targets = { + 'test': 'http://username:password@localhost:5555/testdrive', + } +``` + +In order to specify more options, please wrap your ntfy URL into a dictionary +under the `url` key. This way, additional options can be added. +```ini +[config:ntfy] +targets = { + 'test': { + 'url': 'https://ntfy.sh/testdrive', + }, + } +``` + +:::{note} +[ntfy publishing options] outlines different ways to marshal data to the ntfy +HTTP API. mqttwarn is using the HTTP PUT method, where the HTTP body is used +for the attachment file, and HTTP headers are used for all other ntfy option +fields, encoded with [RFC 2047] MIME [quoted-printable encoding]. +::: + +{#ntfy-remote-attachments} +#### Remote attachments +In order to submit notifications with an attachment file at a remote location, +use the `attach` field. Optionally, the `filename` field can be used to assign +a different name to the file. +```ini +[config:ntfy] +targets = { + 'test': { + 'url': 'https://ntfy.sh/testdrive', + 'attach': 'https://unsplash.com/photos/spdQ1dVuIHw/download?w=320', + 'filename': 'goat.jpg', + }, + } +``` + +{#ntfy-local-attachments} +#### Local attachments +By using the `attachment` option, you can add an attachment to your message, local +to the machine mqttwarn is running on. The file will be uploaded when submitting +the notification, and ntfy will serve it for clients so that you don't have to. In +order to address the file, you can provide a path template, where the transformation +data will also get interpolated into. +```ini +[config:ntfy] +targets = { + 'test': { + 'url': 'https://ntfy.sh/testdrive', + 'file': '/tmp/ntfy-attachment-{slot}-{label}.png', + } + } +``` +:::{important} +In order to allow users to **upload** and attach files to notifications, you will +need to enable the corresponding ntfy feature by simply configuring an attachment +cache directory and a base URL (`attachment-cache-dir`, `base-url`), see +[ntfy stored attachments]. +::: +:::{note} +When mqttwarn processes a message, and accessing the file raises an error, it gets +handled gracefully. In this way, notifications will be triggered even when attaching +the file fails for whatever reasons. +::: + +#### Publishing options +You can use all the available [ntfy publishing options], by using the corresponding +option names listed within `NTFY_FIELD_NAMES`, which are: `message`, `title`, `tags`, +`priority`, `actions`, `click`, `attach`, `filename`, `delay`, `icon`, `email`, +`cache`, `firebase`, and `unifiedpush`. See also the [list of all ntfy option fields]. + +You can obtain ntfy option fields from _three_ contexts in total, as implemented +by the `obtain_ntfy_fields` function. Effectively, that means that you can place +them either within the `targets` address descriptor, within the configuration +section, or submit them using a JSON MQTT message and a corresponding decoder +function into the transformation data dictionary. + +For example, you can always send a `priority` field using MQTT/JSON, or use one of +those configuration snippets, which are equivalent. +```ini +[config:ntfy] +targets = { + 'test': { + 'url': 'https://ntfy.sh/testdrive', + 'priority': 'high', + } + } +``` +```ini +[config:ntfy] +targets = { + 'test': { + 'url': 'https://ntfy.sh/testdrive', + } + } +priority = high +``` + +The highest precedence takes data coming in from the transformation data dictionary, +followed by option fields coming in from the per-recipient `targets` address descriptor, +followed by option fields defined on the `[config:ntfy]` configuration section. + +#### Examples + +1. This is another way to write the "[remote attachments](#ntfy-remote-attachments)" + example, where all ntfy options are located on the configuration section, so they + will apply for all configured target addresses. + ```ini + [config:ntfy] + targets = {'test': 'https://ntfy.sh/testdrive'} + attach = https://unsplash.com/photos/spdQ1dVuIHw/download?w=320 + filename = goat.jpg + ``` + +2. The tutorial [](#processing-frigate-events) explains how to configure mqttwarn to + notify the user with events emitted by Frigate, a network video recorder (NVR) + with realtime local object detection for IP cameras. + +[list of all ntfy option fields]: https://docs.ntfy.sh/publish/#list-of-all-parameters [ntfy]: https://ntfy.sh/ +[ntfy publishing options]: https://docs.ntfy.sh/publish/ +[ntfy stored attachments]: https://docs.ntfy.sh/config/#attachments +[pub-sub]: https://en.wikipedia.org/wiki/Publish%E2%80%93subscribe_pattern +[quoted-printable encoding]: https://en.wikipedia.org/wiki/Quoted-printable +[RFC 2047]: https://datatracker.ietf.org/doc/html/rfc2047 ### `desktopnotify` diff --git a/mqttwarn/commands.py b/mqttwarn/commands.py index 6aaf2bec..1263490a 100644 --- a/mqttwarn/commands.py +++ b/mqttwarn/commands.py @@ -26,7 +26,7 @@ def run(): Usage: {program} [make-config] {program} [make-udf] - {program} [--config=] [--config-file=] [--plugin=] [--options=] + {program} [--config=] [--config-file=] [--plugin=] [--options=] [--data=] {program} --version {program} (-h | --help) @@ -39,8 +39,8 @@ def run(): [--plugin=] The plugin name to load. This can either be a full qualified Python package/module name or a path to a Python file. - [--options=] Configuration options to propagate to the plugin - entrypoint. + [--options=] Configuration options to propagate to the plugin entrypoint. + [--data=] Data to propagate to the plugin entrypoint. Bootstrapping options: make-config Dump configuration file blueprint to STDOUT, @@ -76,14 +76,15 @@ def run(): # Decode arguments arg_plugin = options["--plugin"] - arg_options = json.loads(options["--options"]) + arg_options = options["--options"] and json.loads(options["--options"]) or {} + arg_data = options["--data"] and json.loads(options["--data"]) or {} arg_config = None if "--config" in options and options["--config"] is not None: arg_config = json.loads(options["--config"]) # Launch service plugin in standalone mode launch_plugin_standalone( - arg_plugin, arg_options, configfile=options.get("--config-file"), config_more=arg_config + arg_plugin, arg_options, arg_data, configfile=options.get("--config-file"), config_more=arg_config ) # Run mqttwarn in service mode when no command line arguments are given @@ -91,7 +92,7 @@ def run(): run_mqttwarn() -def launch_plugin_standalone(plugin, options, configfile=None, config_more=None): +def launch_plugin_standalone(plugin, options, data, configfile=None, config_more=None): # Optionally load configuration file does_not_exist = False @@ -120,7 +121,7 @@ def launch_plugin_standalone(plugin, options, configfile=None, config_more=None) logger.info('Running service plugin "{}" with options "{}"'.format(plugin, options)) # Launch service plugin - run_plugin(config=config, name=plugin, options=options) + run_plugin(config=config, name=plugin, options=options, data=data) def run_mqttwarn(): diff --git a/mqttwarn/core.py b/mqttwarn/core.py index 722af4b5..5e9de9eb 100644 --- a/mqttwarn/core.py +++ b/mqttwarn/core.py @@ -784,7 +784,7 @@ def bootstrap(config=None, scriptname=None): SCRIPTNAME = scriptname -def run_plugin(config=None, name=None, options=None): +def run_plugin(config=None, name=None, options=None, data=None): """ Run service plugins directly without the dispatching and transformation machinery. @@ -817,7 +817,7 @@ def run_plugin(config=None, name=None, options=None): item.config = config.config("config:" + name) item.service = srv item.target = "mqttwarn" - item.data = {} # FIXME + item.data = data or {} # Launch plugin module = service_plugins[name]["module"] diff --git a/mqttwarn/model.py b/mqttwarn/model.py index f2455b10..f705bad1 100644 --- a/mqttwarn/model.py +++ b/mqttwarn/model.py @@ -40,7 +40,7 @@ def enum(self): # Covering old- and new-style configuration layouts. `addrs` has # originally been a list of strings, has been expanded to be a # list of dictionaries (Apprise), and to be a dictionary (Pushsafer). -addrs_type = Union[List[Union[str, Dict[str, str]]], Dict[str, str]] +addrs_type = Union[List[Union[str, Dict[str, str]]], Dict[str, str], str] @dataclass @@ -52,6 +52,7 @@ class ProcessorItem: service: Optional[str] = None target: Optional[str] = None config: Dict = field(default_factory=dict) + # TODO: `addrs` can also be a string or dictionary now. addrs: addrs_type = field(default_factory=list) # type: ignore[assignment] priority: Optional[int] = None topic: Optional[str] = None diff --git a/mqttwarn/services/apprise_multi.py b/mqttwarn/services/apprise_multi.py index 4847b017..ea036ba3 100644 --- a/mqttwarn/services/apprise_multi.py +++ b/mqttwarn/services/apprise_multi.py @@ -42,7 +42,7 @@ def plugin(srv, item): # Collect URL parameters. params = OrderedDict() - # Obtain and apply all possible Ntfy parameters from data dictionary. + # Obtain and apply all possible Apprise parameters from data dictionary. params.update(obtain_apprise_arguments(item, APPRISE_ALL_ARGUMENT_NAMES)) # Apply addressee information. diff --git a/mqttwarn/services/apprise_single.py b/mqttwarn/services/apprise_single.py index 7a8ae482..dae8bf4c 100644 --- a/mqttwarn/services/apprise_single.py +++ b/mqttwarn/services/apprise_single.py @@ -43,7 +43,7 @@ def plugin(srv, item): # Collect URL parameters. params = OrderedDict() - # Obtain and apply all possible Ntfy parameters from data dictionary. + # Obtain and apply all possible Apprise parameters from data dictionary. params.update(obtain_apprise_arguments(item, APPRISE_ALL_ARGUMENT_NAMES)) # Apply addressee information. diff --git a/mqttwarn/services/apprise_util.py b/mqttwarn/services/apprise_util.py index e8693bf2..ab4a83c8 100644 --- a/mqttwarn/services/apprise_util.py +++ b/mqttwarn/services/apprise_util.py @@ -30,8 +30,6 @@ def get_all_template_argument_names(): def obtain_apprise_arguments(item: ProcessorItem, arg_names: list) -> dict: """ Obtain eventual Apprise parameters from data dictionary. - - https://github.com/caronc/apprise/wiki/Notify_ntfy#parameter-breakdown """ params = dict() for arg_name in arg_names: diff --git a/mqttwarn/services/ntfy.py b/mqttwarn/services/ntfy.py new file mode 100644 index 00000000..1f6c1d8f --- /dev/null +++ b/mqttwarn/services/ntfy.py @@ -0,0 +1,288 @@ +__author__ = "Andreas Motl " +__copyright__ = "Copyright 2023 Andreas Motl" +__license__ = "Eclipse Public License - v 1.0 (http://www.eclipse.org/legal/epl-v10.html)" + +import dataclasses +import logging +from collections import OrderedDict +import typing as t +from email.header import Header +from pathlib import Path + +import requests +from funcy import project + +from mqttwarn.model import Service, ProcessorItem + +DataDict = t.Dict[str, t.Union[str, bytes]] + + +# Field names to be propagated from transformation data to ntfy API. +# +# `topic` will be omitted, and not picked from the transformation +# data, because it contains the MQTT topic already, and would cause +# collisions. The topic is exclusively defined using the `url` field, +# see https://mqttwarn.readthedocs.io/en/latest/notifier-catalog.html#ntfy. +# +# All other ntfy fields are enumerated here. +# https://docs.ntfy.sh/publish/#publish-as-json +# https://docs.ntfy.sh/publish/#list-of-all-parameters + +NTFY_FIELD_NAMES: t.List[str] = [ + # "topic", + "message", + "title", + "tags", + "priority", + "actions", + "click", + "attach", + "filename", + "delay", + "icon", + "email", + "cache", + "firebase", + "unifiedpush", +] + +NTFY_RFC2047_FIELDS: t.List[str] = [ + "message", + "title", + "tags", +] + +logger = logging.getLogger(__name__) + + +# The `requests` session instance, for running HTTP requests. +http = requests.Session() +# TODO: Add mqttwarn version. +http.headers.update({"User-Agent": "mqttwarn"}) + + +@dataclasses.dataclass +class NtfyRequest: + """ + Manage parameters to be propagated to the ntfy HTTP API. + """ + + url: str + options: t.Dict[str, str] + fields: DataDict + attachment_path: t.Optional[str] + attachment_data: t.Optional[t.Union[bytes, t.IO]] + + def to_http_headers(self) -> t.Dict[str, str]: + """ + Provide a variant for `fields` to be submitted as HTTP headers to the ntfy API. + + Python's `http.client` will, according to the HTTP specification, + encode header values using the `latin-1` character set. + + In this spirit, the header transport does not permit any fancy UTF-8 characters + within any field, so they will be replaced with placeholder characters `?`. + """ + return dict_with_titles(encode_ntfy_fields(self.fields)) + + +def plugin(srv: Service, item: ProcessorItem) -> bool: + """ + mqttwarn service plugin for ntfy. + """ + + srv.logging.debug("*** MODULE=%s: service=%s, target=%s", __file__, item.service, item.target) + + # Decode inbound mqttwarn job item into `NtfyRequest`. + ntfy_request = decode_jobitem(item) + + # Convert field dictionary to HTTP header dictionary. + headers = ntfy_request.to_http_headers() + srv.logging.debug(f"Headers: {dict(headers)}") + + # Submit request to ntfy HTTP API. + try: + srv.logging.info("Sending notification to ntfy. target=%s, options=%s", item.target, ntfy_request.options) + response = http.put(ntfy_request.url, data=ntfy_request.attachment_data, headers=headers) + response.raise_for_status() + except Exception: + srv.logging.exception("Request to ntfy API failed") + return False + + # Report about ntfy response. + srv.logging.debug(f"ntfy response status: {response}") + srv.logging.debug(f"ntfy response content: {response.content!r}") + + srv.logging.info("Successfully sent message using ntfy") + + return True + + +def decode_jobitem(item: ProcessorItem) -> NtfyRequest: + """ + Decode inbound mqttwarn job item into `NtfyRequest`. + """ + + title = item.title + body = item.message + options: t.Dict[str, str] + + if isinstance(item.addrs, str): + options = {"url": item.addrs} + elif isinstance(item.addrs, dict): + options = item.addrs + else: + raise TypeError(f"Unable to handle `targets` address descriptor data type `{type(item.addrs).__name__}`: {item.addrs}") + + url = options["url"] + attachment_path = options.get("file") + + # Collect ntfy fields. + fields: DataDict = OrderedDict() + + # Obtain and propagate all possible ntfy fields from transformation data. + fields.update(obtain_ntfy_fields(item)) + + # Overwrite title and message explicitly, when not present already. + title and fields.setdefault("title", title) + body and fields.setdefault("message", body) + + # Attach a file, or not. + attachment_data = None + if attachment_path: + attachment_path, attachment_data = load_attachment(attachment_path, item.data) + if attachment_data: + # TODO: Optionally derive attachment file name from title, using `slugify(title)`. + fields.setdefault("filename", Path(attachment_path).name) + + ntfy_request = NtfyRequest( + url=url, + options=options, + fields=fields, + attachment_path=attachment_path, + attachment_data=attachment_data, + ) + + return ntfy_request + + +def obtain_ntfy_fields(item: ProcessorItem) -> DataDict: + """ + Obtain eventual ntfy fields from transformation data. + """ + fields_data = item.data and project(item.data, NTFY_FIELD_NAMES) or {} + fields_addrs = item.addrs and project(item.addrs, NTFY_FIELD_NAMES) or {} + fields_config = item.config and project(item.config, NTFY_FIELD_NAMES) or {} + fields: DataDict = OrderedDict() + fields.update(fields_config) + fields.update(fields_addrs) + fields.update(fields_data) + return fields + + +def load_attachment(path: str, tplvars: t.Optional[DataDict]) -> t.Tuple[str, t.Optional[t.IO]]: + """ + Load attachment file from filesystem gracefully. + """ + data = None + try: + path = path.format(**tplvars or {}) + except: + logger.exception(f"ntfy: Computing attachment file name failed") + if path: + try: + data = open(path, "rb") + except: + logger.exception(f"ntfy: Accessing attachment file failed: {path}") + return path, data + + +def ascii_clean(data: t.Union[str, bytes]) -> str: + """ + Return ASCII-clean variant of input string. + https://stackoverflow.com/a/18430817 + """ + if isinstance(data, bytes): + data = data.decode() + if isinstance(data, str): + return data.encode("ascii", errors="replace").decode() + else: + raise TypeError(f"Unknown data type to compute ASCII-clean variant: {type(data).__name__}") + + +def encode_rfc2047(data: t.Union[str, bytes]) -> str: + """ + Return RFC2047-encoded variant of input string. + + https://docs.python.org/3/library/email.header.html + """ + if isinstance(data, bytes): + data = data.decode() + if isinstance(data, str): + return Header(s=data, charset="utf-8").encode() + else: + raise TypeError(f"Unknown data type to compute ASCII-clean variant: {type(data).__name__}") + + +def dict_ascii_clean(data: DataDict) -> t.Dict[str, str]: + """ + Return dictionary with ASCII-clean keys and values. + """ + + outdata = OrderedDict() + for key, value in data.items(): + key = ascii_clean(key).strip() + value = ascii_clean(value).strip() + outdata[key] = value + return outdata + + +def encode_ntfy_fields(data: DataDict) -> t.Dict[str, str]: + """ + Return dictionary suitable for submitting to the ntfy HTTP API using HTTP headers. + + - The field values for `title`, `message` and `tags` are encoded using RFC 2047, aka. + MIME Message Header Extensions for Non-ASCII Text. + + - The other field values will be stripped from any special characters to be ASCII-clean. + + Appendix + + When using RFC 2047, two encodings are possible. The Python implementation cited below + seems to use the "Q" encoding scheme by default. + + 4.1 The "B" encoding is identical to the "BASE64" encoding defined by RFC 2045. + 4.2 The "Q" encoding is similar to the "Quoted-Printable" content-transfer-encoding + defined in RFC 2045. It is designed to allow text containing mostly ASCII + characters to be decipherable on an ASCII terminal without decoding. + + The Python email package supports the standards RFC 2045, RFC 2046, RFC 2047, and + RFC 2231 in its `email.header` and `email.charset` modules. + + - https://datatracker.ietf.org/doc/html/rfc2047#section-2 + - https://datatracker.ietf.org/doc/html/rfc2047#section-4 + - https://docs.python.org/3/library/email.header.html + """ + + outdata = OrderedDict() + for key, value in data.items(): + key = ascii_clean(key).strip() + if key in NTFY_RFC2047_FIELDS: + value = encode_rfc2047(value) + else: + value = ascii_clean(value) + outdata[key] = value + return outdata + + +def dict_with_titles(data: t.Dict[str, str]) -> t.Dict[str, str]: + """ + Return dictionary with each key title-cased, i.e. uppercasing the first letter. + + >>> {"foo": "bar"} + {"Foo": "bar"} + """ + outdata = OrderedDict() + for key, value in data.items(): + outdata[key.title()] = value + return outdata diff --git a/pyproject.toml b/pyproject.toml index ad2e6c0f..9b213507 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,11 @@ extend-exclude = [ [tool.mypy] ignore_missing_imports = true +files = [ + "mqttwarn/core.py", + "mqttwarn/services/ntfy.py", + "tests/services/test_ntfy.py", +] # ================== @@ -110,7 +115,7 @@ lint = [ {cmd="ruff ."}, {cmd="black --check ."}, {cmd="isort --check ."}, - {cmd="mypy --install-types --non-interactive mqttwarn/core.py"}, + {cmd="mypy --install-types --non-interactive"}, ] test = [ {cmd="pytest"}, diff --git a/setup.py b/setup.py index 6e266650..a9812ef2 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ "requests<3", "future>=0.18.0,<1", "importlib-metadata; python_version<'3.8'", + "funcy<3", ] extras = { diff --git a/tests/conftest.py b/tests/conftest.py index 2edab302..6df2ffe2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ # Import custom fixtures. from mqttwarn.testing.fixtures import mqttwarn_service as srv # noqa:F401 +from tests.fixtures.ntfy import ntfy_service # noqa:F401 @pytest.fixture diff --git a/tests/etc/better-addresses.ini b/tests/etc/better-addresses.ini index bbf97f16..c92656a9 100644 --- a/tests/etc/better-addresses.ini +++ b/tests/etc/better-addresses.ini @@ -57,7 +57,6 @@ module = 'apprise_multi' targets = { 'demo-http' : [ { 'baseuri': 'json://localhost:1234/mqtthook' }, { 'baseuri': 'json://daq.example.org:5555/foobar' } ], 'demo-discord' : [ { 'baseuri': 'discord://4174216298/JHMHI8qBe7bk2ZwO5U711o3dV_js' } ], - 'demo-ntfy' : [ { 'baseuri': 'ntfy://user:password@ntfy.example.org/topic1/topic2' } ], 'demo-mailto' : [ { 'baseuri': 'mailtos://smtp_username:smtp_password@mail.example.org', 'recipients': ['foo@example.org', 'bar@example.org'], @@ -102,6 +101,6 @@ targets = { [apprise-test] topic = apprise/# -targets = apprise:demo-http, apprise:demo-discord, apprise:demo-mailto, apprise:demo-ntfy +targets = apprise:demo-http, apprise:demo-discord, apprise:demo-mailto format = Alarm from {device}: {payload} title = Alarm from {device} diff --git a/tests/fixtures/ntfy.py b/tests/fixtures/ntfy.py index 15816792..8482ff14 100644 --- a/tests/fixtures/ntfy.py +++ b/tests/fixtures/ntfy.py @@ -5,12 +5,12 @@ # license that can be found in the LICENSE file or at # https://opensource.org/licenses/MIT. """ -Provide the `Ntfy`_ API service as a session-scoped fixture to your test +Provide the `ntfy`_ API service as a session-scoped fixture to your test harness. Source: https://docs.ntfy.sh/install/#docker -.. _Ntfy: https://ntfy.sh/ +.. _ntfy: https://ntfy.sh/ """ import docker import pytest @@ -22,7 +22,11 @@ "image": "binwiederhier/ntfy", "version": "latest", "options": { - "command": "serve", + "command": """ + serve + --base-url="http://localhost:5555" + --attachment-cache-dir="/tmp/ntfy-attachments" + """, "publish_all_ports": False, "ports": {"80/tcp": "5555"}, }, @@ -64,7 +68,7 @@ def is_ntfy_running() -> bool: @pytest.fixture(scope="session") def ntfy_service(): - # Gracefully skip spinning up the Docker container if Mosquitto is already running. + # Gracefully skip spinning up the Docker container if ntfy is already running. if is_ntfy_running(): yield "localhost", 5555 return diff --git a/tests/services/test_apprise_multi.py b/tests/services/test_apprise_multi.py index 2cbd6c15..f40d8dff 100644 --- a/tests/services/test_apprise_multi.py +++ b/tests/services/test_apprise_multi.py @@ -17,7 +17,6 @@ def test_apprise_multi_basic_success(apprise_asset, apprise_mock, srv, caplog): addrs=[ {"baseuri": "json://localhost:1234/mqtthook"}, {"baseuri": "json://daq.example.org:5555/foobar"}, - {"baseuri": "ntfy://user:password@ntfy.example.org/topic1/topic2"}, ], title="⚽ Message title ⚽", message="⚽ Notification message ⚽", @@ -29,7 +28,6 @@ def test_apprise_multi_basic_success(apprise_asset, apprise_mock, srv, caplog): call(asset=mock.ANY), call().add("json://localhost:1234/mqtthook"), call().add("json://daq.example.org:5555/foobar"), - call().add("ntfy://user:password@ntfy.example.org/topic1/topic2"), call().notify(body="⚽ Notification message ⚽", title="⚽ Message title ⚽"), call().notify().__bool__(), ] @@ -38,8 +36,7 @@ def test_apprise_multi_basic_success(apprise_asset, apprise_mock, srv, caplog): assert ( "Sending notification to Apprise. target=None, addresses=[" "{'baseuri': 'json://localhost:1234/mqtthook'}, " - "{'baseuri': 'json://daq.example.org:5555/foobar'}, " - "{'baseuri': 'ntfy://user:password@ntfy.example.org/topic1/topic2'}" + "{'baseuri': 'json://daq.example.org:5555/foobar'}" "]" in caplog.messages ) assert "Successfully sent message using Apprise" in caplog.messages diff --git a/tests/services/test_apprise_ntfy.py b/tests/services/test_apprise_ntfy.py new file mode 100644 index 00000000..d882927b --- /dev/null +++ b/tests/services/test_apprise_ntfy.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# (c) 2021-2023 The mqttwarn developers +from unittest import mock +from unittest.mock import call + +from mqttwarn.model import ProcessorItem as Item +from mqttwarn.util import load_module_by_name + + +@mock.patch("apprise.Apprise", create=True) +@mock.patch("apprise.AppriseAsset", create=True) +def test_apprise_ntfy_success(apprise_asset, apprise_mock, srv, caplog): + module = load_module_by_name("mqttwarn.services.apprise_multi") + + item = Item( + addrs=[ + { + "baseuri": "ntfy://user:password@ntfy.example.org/topic1/topic2?email=test@example.org", + } + ], + title="⚽ Message title ⚽", + message="⚽ Notification message ⚽", + data={"priority": "high", "tags": "foo,bar", "click": "https://httpbin.org/headers"}, + ) + + outcome = module.plugin(srv, item) + + assert apprise_mock.mock_calls == [ + call(asset=mock.ANY), + call().add( + "ntfy://user:password@ntfy.example.org/topic1/topic2?email=test@example.org" + "&click=https%3A%2F%2Fhttpbin.org%2Fheaders&priority=high&tags=foo%2Cbar" + ), + call().notify(body="⚽ Notification message ⚽", title="⚽ Message title ⚽"), + call().notify().__bool__(), + ] + + assert "Successfully sent message using Apprise" in caplog.messages + assert outcome is True diff --git a/tests/services/test_ntfy.py b/tests/services/test_ntfy.py index 10965775..a8b79540 100644 --- a/tests/services/test_ntfy.py +++ b/tests/services/test_ntfy.py @@ -1,39 +1,339 @@ # -*- coding: utf-8 -*- -# (c) 2021-2023 The mqttwarn developers -from unittest import mock -from unittest.mock import call +# (c) 2023 The mqttwarn developers +import io +import os +import re +import typing as t +from pathlib import Path +from tempfile import NamedTemporaryFile + +import pytest +import responses from mqttwarn.model import ProcessorItem as Item +from mqttwarn.services.ntfy import ( + ascii_clean, + decode_jobitem, + dict_ascii_clean, + dict_with_titles, + encode_rfc2047, + load_attachment, + obtain_ntfy_fields, +) from mqttwarn.util import load_module_by_name -@mock.patch("apprise.Apprise", create=True) -@mock.patch("apprise.AppriseAsset", create=True) -def test_ntfy_success(apprise_asset, apprise_mock, srv, caplog): - module = load_module_by_name("mqttwarn.services.apprise_multi") +@pytest.fixture +def attachment_dummy() -> t.Generator[t.IO[bytes], None, None]: + """ + Provide a temporary files to the test cases to be used as an attachment with defined content. + """ + tmp = NamedTemporaryFile(suffix=".txt", delete=False) + tmp.write(b"foo") + tmp.close() + yield tmp + os.unlink(tmp.name) + + +def test_ntfy_decode_jobitem_overview_success(): + """ + Test the `decode_jobitem` function with a few options. + """ item = Item( - addrs=[ - { - "baseuri": "ntfy://user:password@ntfy.example.org/topic1/topic2?email=test@example.org", - } - ], + addrs={"url": "http://localhost:9999/testdrive"}, title="⚽ Message title ⚽", message="⚽ Notification message ⚽", - data={"priority": "high", "tags": "foo,bar", "click": "https://httpbin.org/headers"}, + data={"priority": "high", "tags": "foo,bar,äöü", "click": "https://example.org/testdrive"}, + ) + + ntfy_request = decode_jobitem(item) + + assert ntfy_request.url == "http://localhost:9999/testdrive" + assert ntfy_request.options["url"] == "http://localhost:9999/testdrive" + assert ntfy_request.fields["message"] == "⚽ Notification message ⚽" + assert ntfy_request.fields["title"] == "⚽ Message title ⚽" + assert ntfy_request.fields["tags"] == "foo,bar,äöü" + assert ntfy_request.fields["priority"] == "high" + assert ntfy_request.fields["click"] == "https://example.org/testdrive" + + +def test_ntfy_decode_jobitem_attachment_success(attachment_dummy): + """ + Test the `decode_jobitem` function with an attachment. + """ + + item = Item( + addrs={"url": "http://localhost:9999/testdrive", "file": attachment_dummy.name}, + ) + + ntfy_request = decode_jobitem(item) + + assert ntfy_request.url == "http://localhost:9999/testdrive" + assert ntfy_request.options["url"] == "http://localhost:9999/testdrive" + assert ntfy_request.options["file"] == attachment_dummy.name + assert ntfy_request.fields["filename"] == Path(attachment_dummy.name).name + assert ntfy_request.attachment_data.read() == b"foo" + + +def test_ntfy_decode_jobitem_attachment_failure(caplog): + """ + Test the `decode_jobitem` function with an invalid attachment. + """ + + item = Item( + addrs={"url": "http://localhost:9999/testdrive", "file": "/tmp/mqttwarn-random-unknown"}, + ) + + ntfy_request = decode_jobitem(item) + + assert ntfy_request.url == "http://localhost:9999/testdrive" + assert ntfy_request.options["url"] == "http://localhost:9999/testdrive" + assert ntfy_request.options["file"] == "/tmp/mqttwarn-random-unknown" + assert "filename" not in ntfy_request.fields + assert ntfy_request.attachment_data is None + + assert "ntfy: Accessing attachment file failed: /tmp/mqttwarn-random-unknown" in caplog.messages + + +def test_ntfy_decode_jobitem_attachment_with_filename_success(attachment_dummy): + """ + Test the `decode_jobitem` function with a user-provided `filename` field. + """ + + item = Item( + addrs={"url": "http://localhost:9999/testdrive", "file": attachment_dummy.name}, + data={"filename": "testdrive.txt"}, + ) + + ntfy_request = decode_jobitem(item) + + assert ntfy_request.url == "http://localhost:9999/testdrive" + assert ntfy_request.options["url"] == "http://localhost:9999/testdrive" + assert ntfy_request.options["file"] == attachment_dummy.name + assert ntfy_request.fields["filename"] == "testdrive.txt" + assert ntfy_request.attachment_data.read() == b"foo" + + +def test_ntfy_decode_jobitem_with_url_only_success(): + """ + Test the `decode_jobitem` function when `addrs` is an URL only. + """ + + item = Item(addrs="http://localhost:9999/testdrive") + + ntfy_request = decode_jobitem(item) + + assert ntfy_request.url == "http://localhost:9999/testdrive" + assert ntfy_request.options["url"] == "http://localhost:9999/testdrive" + + +def test_ntfy_decode_jobitem_with_invalid_target_address_descriptor(): + """ + Test the `decode_jobitem` function when `addrs` is of an invalid type. + """ + + item = Item(addrs=None) + with pytest.raises(TypeError) as ex: + decode_jobitem(item) + assert ex.match(re.escape("Unable to handle `targets` address descriptor data type `NoneType`: None")) + + item = Item(addrs=42.42) + with pytest.raises(TypeError) as ex: + decode_jobitem(item) + assert ex.match(re.escape("Unable to handle `targets` address descriptor data type `float`: 42.42")) + + +def test_ntfy_obtain_ntfy_fields_from_transformation_data(): + """ + Test the `obtain_ntfy_fields` function with transformation data. + + Verify it does not emit fields unknown to ntfy. Here: `garbage`. + """ + indata = {"message": "⚽ Notification message ⚽", "priority": "high", "garbage": "foobar"} + item = Item(data=indata) + outdata = obtain_ntfy_fields(item) + assert list(outdata.keys()) == ["message", "priority"] + + +def test_ntfy_obtain_ntfy_fields_from_config(): + """ + Verify `obtain_ntfy_fields` also obtains data from the configuration section. + """ + indata = {"message": "⚽ Notification message ⚽", "priority": "high", "garbage": "foobar"} + item = Item(config=indata) + outdata = obtain_ntfy_fields(item) + assert list(outdata.keys()) == ["message", "priority"] + + +def test_ntfy_obtain_ntfy_fields_from_options(): + """ + Verify `obtain_ntfy_fields` also obtains data from the target options (addrs). + """ + indata = {"message": "⚽ Notification message ⚽", "priority": "high", "garbage": "foobar"} + item = Item(addrs=indata) + outdata = obtain_ntfy_fields(item) + assert list(outdata.keys()) == ["message", "priority"] + + +def test_ntfy_obtain_ntfy_fields_precedence(): + """ + Verify precedence handling of `obtain_ntfy_fields` when obtaining the same fields from multiple sources. + """ + item = Item(config={"message": "msg-config"}, addrs={"message": "msg-addrs"}, data={"message": "msg-data"}) + outdata = obtain_ntfy_fields(item) + assert outdata["message"] == "msg-data" + + item = Item(config={"message": "msg-config"}, addrs={"message": "msg-addrs"}) + outdata = obtain_ntfy_fields(item) + assert outdata["message"] == "msg-addrs" + + item = Item(config={"message": "msg-config"}) + outdata = obtain_ntfy_fields(item) + assert outdata["message"] == "msg-config" + + +def test_ntfy_load_attachment_tplvar_failure(caplog): + """ + Check how the `load_attachment` helper function fails when the template variables are invalid. + """ + path, data = load_attachment(None, None) + + assert path is None + assert data is None + + assert "ntfy: Computing attachment file name failed" in caplog.messages + assert "AttributeError: 'NoneType' object has no attribute 'format'" in caplog.text + + +def test_ntfy_dict_with_titles(): + """ + Test the `dict_with_titles` helper function. + """ + indata = {"foo": "bar"} + outdata = {"Foo": "bar"} + assert dict_with_titles(indata) == outdata + + +def test_ntfy_dict_ascii_clean(): + """ + Test the `dict_ascii_clean` helper function. + """ + indata = {"message": "⚽ Notification message ⚽", "foobar": "äöü"} + outdata = dict_ascii_clean(indata) + assert outdata["message"] == "? Notification message ?" + assert outdata["foobar"] == "???" + + +def test_ntfy_ascii_clean_success(): + """ + Test the `ascii_clean` helper function. + """ + assert ascii_clean("⚽ Notification message ⚽") == "? Notification message ?" + assert ascii_clean("⚽ Notification message ⚽".encode("utf-8")) == "? Notification message ?" + + +def test_ntfy_encode_rfc2047(): + """ + Test the `ascii_clean` helper function. + """ + message_in = "⚽ Notification message ⚽" + message_out = "=?utf-8?q?=E2=9A=BD_Notification_message_=E2=9A=BD?=" + assert encode_rfc2047(message_in) == message_out + assert encode_rfc2047(message_in.encode("utf-8")) == message_out + + +def test_ntfy_ascii_clean_failure(): + """ + Test the `ascii_clean` helper function. + """ + with pytest.raises(TypeError) as ex: + ascii_clean(None) + assert ex.match(re.escape("Unknown data type to compute ASCII-clean variant: NoneType")) + + +@responses.activate +def test_ntfy_plugin_success(srv, caplog, attachment_dummy): + """ + Test the whole plugin with a successful outcome. + """ + + ntfy_api_response = { + "id": "jBXrDQF4e8ab", + "time": 1681939903, + "expires": 1681983103, + "event": "message", + "topic": "frigate-test", + "title": "goat entered lawn at 2023-04-06 14:31:46.638857+00:00", + "message": "goat was in barn before", + "click": "https://frigate.local/events?camera=cam-testdrive\\u0026label=goat\\u0026zone=lawn", + "attachment": { + "name": "mqttwarn-frigate-cam-testdrive-goat.png", + "type": "image/png", + "size": 283595, + "expires": 1681950703, + "url": "http://localhost:5555/file/jBXrDQF4e8ab.png", + }, + } + + responses.add( + responses.PUT, + "http://localhost:9999/testdrive", + json=ntfy_api_response, + status=200, + ) + + module = load_module_by_name("mqttwarn.services.ntfy") + + item = Item( + addrs={"url": "http://localhost:9999/testdrive", "file": attachment_dummy.name}, + title="⚽ Message title ⚽", + message="⚽ Notification message ⚽", + data={ + "priority": "high", + "tags": "foo,bar,äöü", + "click": "https://example.org/testdrive", + "actions": "view, Adjust temperature 🌡, https://example.org/home-automation/temperature, body='{\"temperature\": 18}'", # noqa: E501 + }, ) outcome = module.plugin(srv, item) - assert apprise_mock.mock_calls == [ - call(asset=mock.ANY), - call().add( - "ntfy://user:password@ntfy.example.org/topic1/topic2?email=test@example.org" - "&click=https%3A%2F%2Fhttpbin.org%2Fheaders&priority=high&tags=foo%2Cbar" - ), - call().notify(body="⚽ Notification message ⚽", title="⚽ Message title ⚽"), - call().notify().__bool__(), - ] - - assert "Successfully sent message using Apprise" in caplog.messages + assert "Successfully sent message using ntfy" in caplog.messages assert outcome is True + + assert len(responses.calls) == 1 + response = responses.calls[0] + assert response.request.url == "http://localhost:9999/testdrive" + assert isinstance(response.request.body, io.BufferedReader) + assert response.request.body.read() == b"foo" + assert response.request.headers["User-Agent"] == "mqttwarn" + assert response.request.headers["Tags"] == "=?utf-8?q?foo=2Cbar=2C=C3=A4=C3=B6=C3=BC?=" + assert ( + response.request.headers["Actions"] + == "view, Adjust temperature ?, https://example.org/home-automation/temperature, body='{\"temperature\": 18}'" + ) + + assert response.response.status_code == 200 + assert response.response.json() == ntfy_api_response + + assert "Successfully sent message using ntfy" in caplog.messages + + +def test_ntfy_plugin_api_failure(srv, caplog): + """ + Processing a message without an ntfy backend API should fail. + """ + + module = load_module_by_name("mqttwarn.services.ntfy") + + item = Item( + addrs={"url": "http://localhost:9999/testdrive"}, + title="⚽ Message title ⚽", + message="⚽ Notification message ⚽", + ) + + outcome = module.plugin(srv, item) + + assert outcome is False + assert "Request to ntfy API failed" in caplog.messages diff --git a/tests/util.py b/tests/util.py index 57a53af3..bddf0741 100644 --- a/tests/util.py +++ b/tests/util.py @@ -2,6 +2,7 @@ # (c) 2018-2021 The mqttwarn developers import shlex import threading +import time from unittest.mock import patch import paho @@ -65,7 +66,8 @@ def mqtt_process(mqttc: paho.mqtt.client.Client, loops=2): """ delay() for _ in range(loops): - mqttc.loop() + mqttc.loop(max_packets=10) + time.sleep(0.01) delay()