From 680688b2f32dd0eeb0a6ff00030b807d8577afd6 Mon Sep 17 00:00:00 2001 From: mmaney Date: Sat, 1 Aug 2020 18:19:56 -0500 Subject: [PATCH] Documentation; JSON catalogs; cli.py cleanup [some] Add catalog.json, replacing ad-hoc lists of providers in cli.py & setup.py. catalog.py for easy use of the catalog, including loading known providers. Add sewer.json info replaces __version__.py; sewer_about in lib to access it. --- docs/CHANGELOG.md | 9 + docs/catalog.md | 142 +++++++++ docs/dns-01.md | 6 +- docs/drivers/route53.md | 17 ++ docs/drivers/unbound_ssh.md | 22 +- docs/notes/0.8.3-notes.md | 36 ++- setup.py | 77 ++--- sewer/__version__.py | 7 - sewer/catalog.json | 202 +++++++++++++ sewer/catalog.py | 78 +++++ sewer/cli.py | 274 ++++++++---------- sewer/client.py | 82 +----- sewer/dns_providers/tests/test_unbound_ssh.py | 75 +++++ sewer/lib.py | 21 +- sewer/sewer.json | 9 + sewer/tests/test_catalog.py | 26 ++ sewer/tests/test_lib.py | 4 + 17 files changed, 793 insertions(+), 294 deletions(-) create mode 100644 docs/catalog.md create mode 100644 docs/drivers/route53.md delete mode 100644 sewer/__version__.py create mode 100644 sewer/catalog.json create mode 100644 sewer/catalog.py create mode 100644 sewer/dns_providers/tests/test_unbound_ssh.py create mode 100644 sewer/sewer.json create mode 100644 sewer/tests/test_catalog.py diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 13be3518..34f07e84 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -13,6 +13,7 @@ Features and Improvements: - added `--p_opt =` for passing kwargs to drivers - Added optional parameters accepted by base class for DNS drivers: - `alias=` specifies a separate domain for DNS challenges + (requires driver support, see [Aliasing](Aliasing)) - `prop_delay=` gives a fixed delay (sleep) after challenge setup - gandi (legacy DNS driver) fixed internal bugs that broke common wildcard use cases (eg., `*.domain.tld`) as well as the "wildcard plus" pattern @@ -21,6 +22,12 @@ Features and Improvements: could be useful to someone, maybe. Internals: +- added [catalog.py](catalog) to manage provider catalogs; includes + get_provider(name) method to replace `import ......{name.}ClassName` +- replace __version__.py with sewer.json; setup.py converted; add sewer_about() + in lib.py; cli.py converted; client.py converted +- added catalog.json defining known drivers and their interfaces; also + information about dependencies for setup.py - added `**kwargs` to all legacy providers to allow new options that are handled in a parent class to pass through (for `alias`, `prop_delay`, etc.) - removed imports that were in `sewer/__init__` and @@ -31,6 +38,8 @@ Internals: - fixed imports in client.py that didn't actually import the parts of OpenSSL and cryptography that we use (worked because we import requests?) +See also [release notes](notes/0.8.3-notes). + ## **version:** 0.8.2 Feature additions: diff --git a/docs/catalog.md b/docs/catalog.md new file mode 100644 index 00000000..5dc65a33 --- /dev/null +++ b/docs/catalog.md @@ -0,0 +1,142 @@ +## Driver Catalog + +The driver catalog, `sewer/catalog.json`, replaces scattered facilities that +were used to stitch things together. The import farms in `sewer.__init__` +and `sewer.dns_providers.__init__` have already been removed; with the +catalog in place, redundant lists in cli.py and setup.py are also removed, +replaced by use of the catalog's data and a few lines of code. The +`dns_provider_name` is deprecated in favor of the catalog as well. + +`catalog.py` wraps the catalog in a class, adding some convenience methods +for listing the known drivers, looking up a driver's data by name, and +loading a driver's implementation class by name. But using the catalog +without `catalog.py` is as easy as loading it using the standard lib's json +facilities - it's all lists & dicts (see eg. setup.py which loads the +catalog this way to avoid potential issues with trying to call into the +package's code before it's installed). + +### Catalog structure + +The catalog resides in a JSON file that loads as an array of dictionaries, +one element for each registered driver. The per-driver record contains the +following items (some optional): + +- **name** The name used to identify this driver, eg., `--provider ` + to `sewer-cli`. These names need to be unique, but are not required to + match the module or implementing class names. (legacy DNS drivers usually + matched the module name, but not always) +- **desc** A brief description of the driver, intended for display to humans + to help them understand what each driver is, eg. in --known_providers output +- **chals** list of strings for the type of challenge(s) this driver + handles. (if more than one type, in order of preference?) +- **args** A list of the driver-specific [parameters](#args-parameter-descriptors) +- **path** The path to use to import the driver's Python module. + _Default_ is `sewer.providers.{name}` +- **cls** Name of module attribute which is called with parameters to get a + working instance of the driver. Usually a class, but a factory function + may be used. _Default_ is `Provider`. +- **features** - a list of strings that name the optional features that this + driver supports. _Default_ is an empty list. +- **memo** Additional text/comments about the driver, the descriptor, etc. +- **deps** list of additional projects this driver requires (for setup) + +### args - parameter desciptors + +This is a bit of a mess due to legacy drivers that ignored the established +conventions. To be fair to them, those conventions weren't clearly +documented (then - see below). This adds some complications to preserve +compatibility, as usual. Let's begin with a minimal descriptor for a driver +that conforms to the new convention (hint: it's imaginary at this time): + + { + "name": "well_behaved", + "desc": "made-up example driver that's mostly defaults", + "chals": ["dns-01"], + "args": [ + { "name": "api_id", "req": 1 }, + { "name": "api_key", "req": 1}, + ], + "features": [ "alias" ], + } + +This describes a dns-01 challenge driver that is found in the module +`sewer.providers.well_behaved`, constructed from a class named `Provider`. +The constructor takes two required arguments, `api_id` and `api_key`, which +the program should accept from environment variables `WELL_BEHAVED_API_ID" +and "WELL_BEHAVED_API_KEY". Since it is a `dns-01` challenge provider and +up to date, it adds the claim that it supports the `alias` feature. It +doesn't support the "unpropagated" feature - perhaps the DNS service has no +API to check the propagation of changes. + +If this had been a difficult old legacy driver, the descriptor might have +looked more like this: + + { + "name": "difficult", + "desc": "made-up example driver that's as non-default as can be", + "chals": ["dns-01"], + "args": [ + { "name": "api_id", + "req": 1, + "param": "difficult_api_id", + "envvar": "DIFFICULT_DNS_API_ID", + }, + { "name": "api_key", + "req": 1, + "param": "DIFFICULT_API_KEY", + "envvar": "DIFFICULT_DNS_API_KEY"}, + }, + { "name": "api_base_url", + "param": "API_BASE_URL", + "envvar": "", + }, + ], + "path": "sewer.dns_providers.difficultdns", + "cls": "DifficultDNSDns", + "features": [], + "memo", "difficult, indeed..." + } + +This driver has both parameter names and envvar names that defy convention, +so both the parameter and envvar name must be given explicitly. There is +also an optional parameter that has never had an associated envvar that the +implementation used. + +### driver parameter and environment variable names + +The convention is that the envvar name (if any) SHOULD be formed from the +driver name and the individual args' names (see the first envvar rule +below). This gives envvar names similar to, sometimes identical to, the +ones already used with legacy DNS drivers. One thing that is changing is +that the parameter names, which in the old convention were +THE_SAME_AS_ENVVAR_NAMES, are changing to be lower case and losing +driver-name prefixes, etc. Where appropriate, the new names will use just a +few shared names, viz., `id`, `key`, `token`. + +Obviously the drivers and envvar names are not so consistent among the +legacy DNS drivers. Therefore the descriptor has both `param` and `envvar` +values, along with a set of rules for resolving the names to be used. + +#### parameter name rules + +1. `descriptor.args[n].name` is the "modern" name for the nth parameter +2. if `param` is given, it overrides the "modern" name + +#### environment name rules + +1. f"{descriptor.name}_{descriptor.args[n].name}".upper() is the default +2. if `envvar` is given, it overrides the default + +Two guidelines for the use of envvars: + +1. If `envvar` is given, is not the empty string, and the so-named envvar is + not found, the invoking code MAY also look for the default-named envvar + before reporting a missing envvar. + +2. If `envvar` is set to the empty string, then catalog using code will not + look for a matching envvar at all. + +### catalog representation in Python + +For now, see the brief implementation in sewer/catalog.py for the way the +JSON structure is mapped into a ProviderDescriptor instance. diff --git a/docs/dns-01.md b/docs/dns-01.md index 5f3104c3..2cda85be 100644 --- a/docs/dns-01.md +++ b/docs/dns-01.md @@ -19,6 +19,7 @@ service's API is difficult. - [Hurricane Electric DNS](https://dns.he.net/) - [PowerDNS](https://doc.powerdns.com/authoritative/http-api/index.html) - [Rackspace](https://www.rackspace.com/cloud/dns) +- [unbound_ssh] ### Add a driver for your DNS service @@ -95,9 +96,12 @@ Three features that have varying support in the Legacy drivers. | hurricane | ? | no | no | test coverage 70% | | powerdns | NO | no | no | apparently not in 0.8.2; bug #195 | | rackspace | ? | no | no | test coverage 69% | -| route53 | OK | no | no | wildcard since pre-0.8.2 | +| route53 (1) | OK | no | no | wildcard since pre-0.8.2 | | unbound_ssh | OK | yes | no | Working demonstrator model | +> (1) route53 was never setup to be used from `sewer-cli`. That will change, +maybe for 0.8.3, but does anyone care? No complaints have been heard... + _wildcard_ is NOT the older issue - since 0.8.2, all drivers should be able to support creating certificates for simple `*.domain.tld` requests. There is a deeper problem when one wants a wildcard that _also_ covers plain diff --git a/docs/drivers/route53.md b/docs/drivers/route53.md new file mode 100644 index 00000000..25ac692a --- /dev/null +++ b/docs/drivers/route53.md @@ -0,0 +1,17 @@ +## route53 - driver for AWS DNS service + +### Command line use + +route53 has never been wired into `sewer-cli`, and that hasn't really +changed in 0.8.3. It does appear in the list of "known providers", but it +isn't usable, and raises an exception if named by `--provider`. + +Adding that integration is on the list, but seeing as no one has complained +about this lack up to now it's nowhere near the top. :-( + +### Programmatic use + +Apparently everyone using sewer's route53 has been rolling their own +wrapper, since it has only been available for such use to date. There is a +patch to extend that Route53Dns.__init__ to allow additional AWS-specific +methods of authentication which I expect will ship in 0.8.3. diff --git a/docs/drivers/unbound_ssh.md b/docs/drivers/unbound_ssh.md index 0d89880b..78a60f30 100644 --- a/docs/drivers/unbound_ssh.md +++ b/docs/drivers/unbound_ssh.md @@ -1,17 +1,21 @@ ## unbound_ssh legacy DNS driver -A working, if somewhat quirky, driver to setup challenges in the unbound -server, using ssh to connect to an account able to run unbound-control -commands. The driver does NOT handle the login authorization, assuming that -it is running interactively and ssh will prompt for your input, or that a -key agent (eg., ssh-agent) is active to supply the cryptographic -credentials. +A working, if somewhat quirky, driver to setup challenges in local data of +the [unbound](https://nlnetlabs.nl/projects/unbound/about/) caching +resolver. As the name suggests, it relies upon ssh to provide an +authenticated connection the server; inside that connection the +`unbound-control` program is used to add and remove the records. The driver +does NOT handle the login authorization, assuming that it is running +interactively and ssh will prompt for your input, or that a key agent (eg., +ssh-agent) is active to supply the cryptographic credentials. That's the +_somewhat quirky_ part! ### `__init__(self, *, ssh_des, **kwargs)` There is one REQUIRED parameter, `ssh_des`, which is the login target, such as acme_user@ns1.example.com. This is simply passed to the ssh command, -with the unbound-control commands passed as the command to execute remotely. +along with the `unbound-control` commands to be executed on the destination +machine. ### Driver features @@ -38,3 +42,7 @@ Sadly, This was written using the old paradigm where both the module name and the class name were more-or-less the same name aside from capitalization... and often less predictable changes. Should have been unbound_ssh.Provider ... + +The `unbound-control` commands generated could be run locally with not very +much change to the driver. Perhaps that will become part of a demonstration +of some different features in the future. diff --git a/docs/notes/0.8.3-notes.md b/docs/notes/0.8.3-notes.md index 52cc7121..ec691c01 100644 --- a/docs/notes/0.8.3-notes.md +++ b/docs/notes/0.8.3-notes.md @@ -1,4 +1,4 @@ -# Sewer 0.8.3 Release Notes +## Sewer 0.8.3 Release Notes This will attempt to list all the changes that affect users of the `sewer-cli` program, including even cosmetic changes. If you use sewer as a @@ -7,7 +7,7 @@ library you may find internal changes not called out here. **New `sewer-cli` features are usually just mentioned, see [sewer-cli.md](sewer-cli) for more complete documentation.** -## What's New +### What's New - added many words in the /docs directory. A lot of it is internals documentation; a lot of it was written to help me understand exactly how @@ -34,7 +34,7 @@ library you may find internal changes not called out here. **NB: `--alias_domain` and the planned `--prop_*` options were only added during PRE-0.8.3, so they will just be dropped in the release.** -## What's Changed +### What's Changed Mostly I've tried to avoid changes that were likely to break things. More so for `sewer-cli` than those who use the inner workings of Client, of @@ -59,10 +59,34 @@ course. a legacy DNS driver. Needs a rather specific environment to work, but I just renewed a handful of certificates using it the other day. -## Breakage +- JSON configuration has arrived. sewer.json replaces __version__.py. + catalog.json adds a central description of known drivers, replacing the + mess of imports [removed], some non-DRY lists in setup.py and cli.py. + +- `sewer.catalog` provides loading of the JSON catalog as well as methods to + lookup the descriptor and load the module; replaces the mess of imports + +### Breakage - removed all the imports in __init__.py (both sewer & dns_providers). This *will* affect you if you've just done `import sewer` and access especially - the provider classes as eg. `sewer.ThatDNSDns`. Using proper imports, + the provider classes as eg. `sewer.ThatDNSDns`. ~~Using proper imports, eg., `import sewer.dns_providers.thatdns.ThatDNSDns` is the current - workaround, sorry. **This does NOT affect `sewer-cli` users.** + workaround, sorry.~~ Recommended use is now something like + ``` + from sewer import catalog + + pro_cls = catalog.ProviderCatalog().get_provider("route53") + provider = pro_cls(...arguments as needed...) + + # use of route53 is ironic: it has no catalog entry because it has + # never been integrated into `sewer-cli`, which was what drove the + # creation of the catalog [right, another FIX ME] + ``` + **This does NOT affect `sewer-cli` users.** + +### Deprecated + +- `--action {run,renew}` option has never actually had any effect and is no + longer required (since 0.8.2?). LOGS A WARNING IN 0.8.3. +- ... diff --git a/setup.py b/setup.py index 43f66c88..ce874838 100644 --- a/setup.py +++ b/setup.py @@ -1,57 +1,47 @@ -import os +import codecs, json, os from setuptools import setup, find_packages -# To use a consistent encoding -import codecs - here = os.path.abspath(os.path.dirname(__file__)) -about = {} + +### FIX ME ### endgoal is to have README.rst, linking to komuw.github.io/sewer for most of it +### # # # #### "pandoc delanda est!" try: import pypandoc long_description = pypandoc.convert("docs/README.md", "rst") except ImportError: - long_description = codecs.open("docs/README.md", encoding="utf8").read() + with codecs.open("docs/README.md", "r", encoding="utf8") as f: + long_description = f.read() + + +with codecs.open(os.path.join(here, "sewer", "sewer.json"), "r", encoding="utf8") as f: + about = json.load(f) + +with codecs.open(os.path.join(here, "sewer", "catalog.json"), "r", encoding="utf8") as f: + catalog = json.load(f) -with open(os.path.join(here, "sewer", "__version__.py"), "r") as f: - exec(f.read(), about) +provider_deps_map = dict(dict((i["name"], i["deps"]) for i in catalog)) -dns_provider_deps_map = { - "cloudflare": [""], - "aliyun": ["aliyun-python-sdk-core-v3", "aliyun-python-sdk-alidns"], - "hurricane": ["hurricanedns"], - "aurora": ["tldextract", "apache-libcloud"], - "acmedns": ["dnspython"], - "rackspace": ["tldextract"], - "dnspod": [""], - "duckdns": [""], - "cloudns": ["cloudns-api"], - "route53": ["boto3"], - "powerdns": [""], -} +all_deps_of_all_providers = list(set(sum((i["deps"] for i in catalog), []))) -all_deps_of_all_dns_provider = [] -for _, vlist in dns_provider_deps_map.items(): - all_deps_of_all_dns_provider += vlist -all_deps_of_all_dns_provider = list(set(all_deps_of_all_dns_provider)) setup( - name=about["__title__"], + name=about["title"], # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # https://packaging.python.org/en/latest/single_source_version.html - version=about["__version__"], - description=about["__description__"], + version=about["version"], + description=about["description"], long_description=long_description, # The project's main homepage. - url=about["__url__"], + url=about["url"], # Author details - author=about["__author__"], - author_email=about["__author_email__"], + author=about["author"], + author_email=about["author_email"], # Choose your license - license=about["__license__"], + license=about["license"], # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ # How mature is this project? Common values are @@ -95,28 +85,19 @@ # dependencies). You can install these using the following syntax, # for example: # $ pip3 install -e .[dev,test] - extras_require={ - "dev": ["coverage", "pypandoc", "twine", "wheel"], - "test": ["pylint==2.3.1", "black==18.9b0"], - "cloudflare": dns_provider_deps_map["cloudflare"], - "aliyun": dns_provider_deps_map["aliyun"], - "hurricane": dns_provider_deps_map["hurricane"], - "aurora": dns_provider_deps_map["aurora"], - "acmedns": dns_provider_deps_map["acmedns"], - "rackspace": dns_provider_deps_map["rackspace"], - "dnspod": dns_provider_deps_map["dnspod"], - "duckdns": dns_provider_deps_map["duckdns"], - "cloudns": dns_provider_deps_map["cloudns"], - "route53": dns_provider_deps_map["route53"], - "powerdns": dns_provider_deps_map["powerdns"], - "alldns": all_deps_of_all_dns_provider, - }, + extras_require=dict( + provider_deps_map, + dev=["coverage", "pypandoc", "twine", "wheel"], + test=["pylint==2.3.1", "black==18.9b0"], + alldns=all_deps_of_all_providers, + ), # If there are data files included in your packages that need to be # installed, specify them here. If using Python 2.6 or less, then these # have to be included in MANIFEST.in as well. # package_data={ # 'sample': ['package_data.dat'], # }, + package_data={"sewer": ["*.json"]}, # Although 'package_data' is the preferred approach, in some case you may # need to place data files outside of your packages. See: # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa diff --git a/sewer/__version__.py b/sewer/__version__.py deleted file mode 100644 index f2c04450..00000000 --- a/sewer/__version__.py +++ /dev/null @@ -1,7 +0,0 @@ -__title__ = "sewer" -__description__ = "Sewer is a programmatic Lets Encrypt(ACME) client" -__url__ = "https://github.com/komuw/sewer" -__version__ = "PRE-0.8.3" -__author__ = "komuW" -__author_email__ = "komuw05@gmail.com" -__license__ = "MIT" diff --git a/sewer/catalog.json b/sewer/catalog.json new file mode 100644 index 00000000..bf52b234 --- /dev/null +++ b/sewer/catalog.json @@ -0,0 +1,202 @@ +[ + { "name": "acmedns", + "desc": "AcmeDns DNS provider", + "chals": ["dns-01"], + "args": [ + { "name": "api_user", + "req": 1, + "old_param": "ACME_DNS_API_USER" + }, + { "name": "api_key", + "req": 1, + "old_param": "ACME_DNS_API_KEY" + }, + { "name": "api_base_url", + "req": 1, + "old_param": "ACME_DNS_API_BASE_URL" + } + ], + "path": "sewer.dns_providers.acmedns", + "cls": "AcmeDnsDns", + "deps": ["dnspython"] + }, + { "name": "aliyun", + "desc": "Alibaba Cloud DNS service", + "chals": ["dns-01"], + "args": [ + { "name": "ak", + "req": 1, + "old_param": "aliyun_ak", + "old_envvar": "ALIYUN_AK_ID" + }, + { "name": "secret", + "req": 1, + "old_param": "aliyun_secret", + "old_envvar": "ALIYUN_AK_SECRET" + }, + { "name": "endpoint", + "old_param": "aliyun_endpoint", + "old_envvar": "ALIYUN_ENDPOINT" + } + ], + "path": "sewer.dns_providers.aliyundns", + "cls": "AliyunDns", + "deps": ["aliyun-python-sdk-core-v3", "aliyun-python-sdk-alidns"], + "memo": "default value of endpoint in ad-hoc cli.py code - add default to args?" + }, + { "name": "aurora", + "desc": "Aurora DNS service from pcextreme hosting", + "chals": ["dns-01"], + "args": [ + { "name": "api_key", + "req": 1 + }, + { "name": "secret_key", + "req": 1 + } + ], + "path": "sewer.dns_providers.auroradns", + "cls": "AuroraDns", + "deps": ["tldextract", "apache-libcloud"] + }, + { "name": "cloudflare", + "desc": "Cloudflare DNS using either email & key or just a token", + "chals": ["dns-01"], + "args": [ + { "name": "email"}, + { "name": "api_key"}, + { "name": "api_base_url"}, + { "name": "token"} + ], + "path": "sewer.dns_providers.cloudflare", + "cls": "CloudFlareDns", + "deps": [], + "memo": "accepts EITHER token OR both email & key; drive MUST sanity check" + }, + { "name": "cloudns", + "desc": "ClouDNS service", + "chals": ["dns-01"], + "args": [ + ], + "path": "sewer.dns_providers.cloudns", + "cls": "ClouDNSDns", + "deps": ["cloudns-api"], + "memo": "API library grovels the environment for its access parameters directly" + }, + { "name": "dnspod", + "desc": "DNSPod DNS provider", + "chals": ["dns-01"], + "args": [ + { "name": "id", "req": 1}, + { "name": "api_key", "req": 1}, + { "name": "api_base_url"} + ], + "path": "sewer.dns_providers.dnspod", + "cls": "DNSPodDns", + "deps": [], + "memo": "api_base_url not usually used? [VERIFY]" + }, + { "name": "duckdns", + "desc": "DuckDNS DNS provider", + "chals": ["dns-01"], + "args": [ + { "name": "token", + "req": 1, + "old_param": "duckdns_token", + "old_envvar": "DUCKDNS_TOKEN" + }, + { "name": "api_base_url", + "old_param": "DUCKDNS_API_BASE_URL", + "old_envvar": "" + } + ], + "path": "sewer.dns_providers.duckdns", + "cls": "DuckDNSDns", + "deps": [], + "memo": "as-is code does not look for envvar for base_url; maybe for testing only?" + }, + { "name": "gandi", + "desc": "Gandi DNS service", + "chals": ["dns-01"], + "args": [ + { "name": "api_key", + "req": 1, + "old_param": "GANDI_API_KEY" + } + ], + "path": "sewer.dns_providers.gandi", + "cls": "GandiDns", + "deps": [] + }, + { "name": "hurricane", + "desc": "Hurricane Electric DNS service", + "chals": ["dns-01"], + "args": [ + { "name": "username", + "req": 1, + "old_param": "he_username" + }, + { "name": "password", + "req": 1, + "old_param": "he_password" + } + ], + "path": "sewer.dns_providers.hurricane", + "cls": "HurricaneDns", + "deps": ["hurricanedns"] + }, + { "name": "powerdns", + "desc": "PowerDNS DNS provider", + "chals": ["dns-01"], + "args": [ + { "name": "api_key", + "req": 1, + "old_param": "powerdns_api_key", + "old_envvar": "POWERDNS_API_KEY" + }, + { "name": "api_url", + "req": 1, + "old_param": "powerdns_api_url", + "old_envvar": "POWERDNS_API_URL" + } + ], + "path": "sewer.dns_providers.powerdns", + "cls": "PowerDNSDns", + "deps": [], + "memo": "could drop old_envvar if prediction ignored old_param?" + }, + { "name": "rackspace", + "desc": "Rackspace DNS service", + "chals": ["dns-01"], + "args": [ + { "name": "username", "req": 1}, + { "name": "api_key", "req": 1} + ], + "path": "sewer.dns_providers.rackspace", + "cls": "RackspaceDns", + "deps": ["tldextract"] + }, + { "name": "route53", + "desc": "Amazon cloud DNS service", + "chals": ["dns-01"], + "args": [ + { "name": "id", "req": 1, "old_param": "access_key_id"}, + { "name": "key", "req": 1, "old_param": "secret_access_key"} + ], + "path": "sewer.dns_providers.route53", + "cls": "Route53Dns", + "deps": ["boto3"], + "memo": "DUMMY LISTING: route53 has never been integrated into cli.py, so this DOESN'T WORK yet" + }, + { "name": "unbound_ssh", + "desc": "Working demonstrater of legacy DNS adopting new features", + "chals": ["dns-01"], + "args": [ + { "name": "ssh_des", "req": 1, "envvar": "" } + ], + "path": "sewer.dns_providers.unbound_ssh", + "cls": "UnboundSsh", + "features": ["alias"], + "deps": [] + } +] diff --git a/sewer/catalog.py b/sewer/catalog.py new file mode 100644 index 00000000..8c575b25 --- /dev/null +++ b/sewer/catalog.py @@ -0,0 +1,78 @@ +import codecs, importlib, json, os, sys +from typing import Dict, List, Sequence + +from .auth import ProviderBase + + +class ProviderDescriptor: + def __init__( + self, + *, + name: str, + desc: str, + chals: Sequence[str], + args: Sequence[Dict[str, str]], + deps: Sequence[str], + path: str = None, + cls: str = None, + features: Sequence[str] = None, + memo: str = None + ) -> None: + "initialize a driver descriptor from one item in the catalog" + + self.name = name + self.desc = desc + self.chals = chals + self.args = args + self.deps = deps + self.path = path + self.cls = cls + self.features = [] if features is None else features + self.memo = memo + + def __str__(self) -> str: + return "Descriptor %s" % self.name + + def get_provider(self) -> ProviderBase: + "return the class that implements this driver" + + module_name = self.path if self.path else ("sewer.providers." + self.name) + module = importlib.import_module(module_name) + return getattr(module, self.cls if self.cls else "Provider") + + +class ProviderCatalog: + def __init__(self, filepath: str = "") -> None: + "intialize a catalog from either the default catalog.json or one named by filepath" + + if not filepath: + here = os.path.abspath(os.path.dirname(__file__)) + filepath = os.path.join(here, "catalog.json") + with codecs.open(filepath, "r", encoding="utf8") as f: + raw_catalog = json.load(f) + + items: Dict[str, ProviderDescriptor] = {} + for item in raw_catalog: + k = item["name"] + if k in items: + print("WARNING: duplicate name %s skipped in catalog %s" % (k, filepath)) + else: + items[k] = ProviderDescriptor(**item) + self.items = items + + def get_item_list(self) -> List[ProviderDescriptor]: + "return the list of items in the catalog, sorted by name" + + res = [i for i in self.items.values()] + res.sort(key=lambda i: i.name) + return res + + def get_descriptor(self, name: str) -> ProviderDescriptor: + "return the ProviderDescriptor that matches name" + + return self.items[name] + + def get_provider(self, name: str) -> ProviderBase: + "return the class that implements the named driver" + + return self.get_descriptor(name).get_provider() diff --git a/sewer/cli.py b/sewer/cli.py index dc8770dc..1516c2ed 100644 --- a/sewer/cli.py +++ b/sewer/cli.py @@ -2,82 +2,55 @@ import argparse from typing import List -from .client import Client -from . import __version__ as sewer_version -from .config import ACME_DIRECTORY_URL_STAGING, ACME_DIRECTORY_URL_PRODUCTION -from .lib import create_logger +from . import client, config, lib +# from .lib import create_logger, sewer_about +from .catalog import ProviderCatalog -### this is a stand-in for the planned (and more disruptive) provider catalog -_known_providers = { - "cloudflare": "CloudFlare DNS service", - "aurora": "AururaDNS service", - "acmedns": "AcmeDNS service", - "aliyun": "Aliyun DNS service", - "hurricane": "Hurricane Electric DNS", - "rackspace": "RackSpace DNS service", - "dnspod": "DNSPod service", - "duckdns": "DuckDNS service", - "cloudns": "ClouDNS service", - "powerdns": "PowerDNS service", - "gandi": "Gandi DNS service", - "unbound_ssh": "Demonstrater to manage unbound through ssh", -} +def setup_parser(catalog): + """ + return configured ArgumentParser - catalog-driven list of providers + """ - -def known_providers_names() -> List[str]: - names = [k for k in _known_providers] - names.sort() - return names - - -def known_providers_list() -> List[str]: - res = [] - for name in known_providers_names(): - res.append("%s: %s" % (name, _known_providers[name])) - return res - - -def setup_parser(): parser = argparse.ArgumentParser( prog="sewer", description="Sewer is an ACME client for getting certificates from Let's Encrypt", allow_abbrev=False, + formatter_class=argparse.RawTextHelpFormatter, ) + + ### immediate action "options" + parser.add_argument( "--version", action="version", - version="%(prog)s {version}".format(version=sewer_version.__version__), + version="%(prog)s {version}".format(version=lib.sewer_about("version")), help="The currently installed sewer version.", ) + parser.add_argument( + "--known_providers", + action="version", + version="Known Providers:\n " + + "\n ".join("%s %s" % (i.name, i.desc) for i in catalog.get_item_list()), + help="Show a list of the known providers and exit.", + ) + + ### ACME account options + parser.add_argument( "--account_key", type=argparse.FileType("r"), - help="Filepath of existing ACME account key to use. Default is to create one.", + help="Filepath to read to get registered ACME account. Default is to create one.", ) + parser.add_argument("--email", help="Email to be used for registration of an ACME account.") + + ### certificate options + parser.add_argument( "--certificate_key", type=argparse.FileType("r"), - help="Filepath to existing certificate key to use. Default is to create one.", - ) - parser.add_argument( - "--provider", - "--dns", - metavar="", - dest="provider", - required=True, - choices=known_providers_names(), - help="Name of the challenge provider to use. (--dns is OBSOLESCENT; prefer --provider)", - ) - parser.add_argument( - "--p_opts", default=[], nargs="*", help="Option(s) to pass to provider, each is key=value" - ) - parser.add_argument( - "--known_providers", - action="version", - version="Known Providers:\n " + "\n ".join(known_providers_list()), - help="Show a list of the known providers and exit.", + help="Filepath to read to get certificate key. Default is to create one.", ) parser.add_argument( "--domain", @@ -90,34 +63,48 @@ def setup_parser(): nargs="*", help="Optional alternate (SAN) identities to be added to the CN on this certificate.", ) - parser.add_argument( - "--alias_domain", help="*** accepted but not implemented through most drivers yet ***" - ) parser.add_argument( "--bundle_name", help="The basename for the output files. Default is the CN given by --domain.", ) + parser.add_argument( + "--out_dir", + default=os.getcwd(), + help="Directory that stores certificate and keys files; current dir is default.", + ) + + ### challenge provider options + + parser.add_argument( + "--provider", + "--dns", + metavar="", + dest="provider", + required=True, + choices=[i.name for i in catalog.get_item_list()], + help="Name of the challenge provider to use. (--dns is OBSOLESCENT; prefer --provider)", + ) + parser.add_argument( + "--p_opts", default=[], nargs="*", help="Option(s) to pass to provider, each is key=value" + ) + + ### protocol options + parser.add_argument( "--endpoint", default="production", choices=["production", "staging"], help="Select between Let's Encrypt's endpoints. Default is production.", ) - parser.add_argument("--email", help="Email to be used for registration of an ACME account.") - parser.add_argument( - "--action", - choices=["run", "renew"], - default="renew", - help="The action that you want to perform. [Obsolescent? Changes nothing but message.]", - ) parser.add_argument( - "--out_dir", - default=os.getcwd(), - help="""The dir where the certificate and keys file will be stored. - default: The directory you run sewer command. - eg: --out_dir /data/ssl/ - """, + "--acme_timeout", + type=int, + default=7, + help="The maximum time the client will wait for a network call (HTTPS request to ACME server) to complete. Default is 7", ) + + ### sewer command options + parser.add_argument( "--loglevel", default="INFO", @@ -126,76 +113,19 @@ def setup_parser(): eg: --loglevel DEBUG", ) parser.add_argument( - "--acme_timeout", - type=int, - default=7, - help="The maximum time the client will wait for a network call (HTTPS request to ACME server) to complete. Default is 7", - ) - parser.add_argument( - "--prop_delay", - type=int, - default=0, - help="Add n second delay for propagation between setup and asking for validation check", + "--action", + choices=["run", "renew"], + default="none", + help="[DEPRECATED] The action that you want to perform (has never done anything).", ) return parser -def main(): - "See docs/sewer-cli.md for docs & examples" - - parser = setup_parser() - args = parser.parse_args() - - loglevel = args.loglevel - logger = create_logger(None, loglevel) - - provider_name = args.provider - domain = args.domain - alt_domains = args.alt_domains - action = args.action - account_key = args.account_key - certificate_key = args.certificate_key - bundle_name = args.bundle_name - endpoint = args.endpoint - email = args.email - out_dir = args.out_dir - - ### FIX ME ### to keep special options --domain_alias & --prop-*, or use -p_opts instead? - - provider_kwargs = {} - if args.alias_domain: - provider_kwargs["alias"] = args.alias_domain - logger.warning( - "--alias_domain is OBSOLETE but accepted during PRE-0.8.3. Use --p_opt alias=... instead" - ) - if args.prop_delay > 0: - provider_kwargs["prop_delay"] = args.prop_delay - logger.warning( - "--prop_delay is OBSOLETE but accepted during PRE-0.8.3. Use --p_opt prop_delay=... instead" - ) - - for p in args.p_opts: - parts = p.split("=") - if len(parts) == 2: - provider_kwargs[parts[0]] = parts[1] - - # Make sure the output dir user specified is writable - if not os.access(out_dir, os.W_OK): - raise OSError("The dir '{0}' is not writable".format(out_dir)) - - if account_key: - account_key = account_key.read() - if certificate_key: - certificate_key = certificate_key.read() - if bundle_name: - file_name = bundle_name - else: - file_name = "{0}".format(domain) - if endpoint == "staging": - ACME_DIRECTORY_URL = ACME_DIRECTORY_URL_STAGING - else: - ACME_DIRECTORY_URL = ACME_DIRECTORY_URL_PRODUCTION +def get_provider(provider_name, provider_kwargs, catalog, logger): + """ + return class (or callable) that will return the Provider instance to use + """ if provider_name == "cloudflare": from .dns_providers.cloudflare import CloudFlareDns @@ -361,10 +291,67 @@ def main(): dns_class = UnboundSsh(**provider_kwargs) # pylint: disable=E1125 logger.info("chosen_provider_name. Using {0} as dns provider.".format(provider_name)) + elif provider_name == "route53": + raise ValueError("route53 driver can only be used programmatically at this time, sorry") + else: raise ValueError("The dns provider {0} is not recognised.".format(provider_name)) - client = Client( + return dns_class + + +def main(): + "See docs/sewer-cli.md for docs & examples" + + catalog = ProviderCatalog() + + parser = setup_parser(catalog) + args = parser.parse_args() + + loglevel = args.loglevel + logger = lib.create_logger(None, loglevel) + + provider_name = args.provider + domain = args.domain + alt_domains = args.alt_domains + if args.action != "none": + logger.warning("DEPRECATION WARNING: --action option is obsolete and will be dropped soon") + account_key = args.account_key + certificate_key = args.certificate_key + bundle_name = args.bundle_name + endpoint = args.endpoint + email = args.email + out_dir = args.out_dir + + ### FIX ME ### to keep special options --domain_alias & --prop-*, or use -p_opts instead? + + provider_kwargs = {} + + for p in args.p_opts: + parts = p.split("=") + if len(parts) == 2: + provider_kwargs[parts[0]] = parts[1] + + # Make sure the output dir user specified is writable + if not os.access(out_dir, os.W_OK): + raise OSError("The dir '{0}' is not writable".format(out_dir)) + + if account_key: + account_key = account_key.read() + if certificate_key: + certificate_key = certificate_key.read() + if bundle_name: + file_name = bundle_name + else: + file_name = "{0}".format(domain) + if endpoint == "staging": + ACME_DIRECTORY_URL = config.ACME_DIRECTORY_URL_STAGING + else: + ACME_DIRECTORY_URL = config.ACME_DIRECTORY_URL_PRODUCTION + + dns_class = get_provider(provider_name, provider_kwargs, catalog, logger) + + acme_client = client.Client( provider=dns_class, domain_name=domain, domain_alt_names=alt_domains, @@ -375,8 +362,8 @@ def main(): LOG_LEVEL=loglevel, ACME_REQUEST_TIMEOUT=args.acme_timeout, ) - certificate_key = client.certificate_key - account_key = client.account_key + certificate_key = acme_client.certificate_key + account_key = acme_client.account_key # prepare file path account_key_file_path = os.path.join(out_dir, "{0}.account.key".format(file_name)) @@ -388,12 +375,7 @@ def main(): account_file.write(account_key) logger.info("account key succesfully written to {0}.".format(account_key_file_path)) - if action == "renew": - message = "Certificate Succesfully renewed. The certificate, certificate key and account key have been saved in the current directory" - certificate = client.renew() - else: - message = "Certificate Succesfully issued. The certificate, certificate key and account key have been saved in the current directory" - certificate = client.cert() + certificate = acme_client.cert() # write out certificate and certificate key in out_dir directory with open(crt_file_path, "w") as certificate_file: @@ -403,5 +385,3 @@ def main(): logger.info("certificate succesfully written to {0}.".format(crt_file_path)) logger.info("certificate key succesfully written to {0}.".format(crt_key_file_path)) - - logger.info("the_end. {0}".format(message)) diff --git a/sewer/client.py b/sewer/client.py index 6aef5a5e..50909331 100644 --- a/sewer/client.py +++ b/sewer/client.py @@ -13,49 +13,14 @@ import cryptography.hazmat.primitives.serialization import cryptography.hazmat.backends -from . import __version__ as sewer_version from .config import ACME_DIRECTORY_URL_PRODUCTION from .auth import ChalListType, ErrataListType -from .lib import create_logger, log_response, safe_base64 +from .lib import create_logger, log_response, safe_base64, sewer_about class Client: """ - todo: improve documentation. - - usage: - import sewer - provider = sewer.CloudFlareDns( - CLOUDFLARE_EMAIL='example@example.com', - CLOUDFLARE_API_KEY='nsa-grade-api-key' - ) - - ### to create a new certificate. - client = sewer.Client( - domain_name='example.com', provider=provider - ) - certificate = client.cert() - certificate_key = client.certificate_key - account_key = client.account_key - - with open('certificate.crt', 'w') as certificate_file: - certificate_file.write(certificate) - - with open('certificate.key', 'w') as certificate_key_file: - certificate_key_file.write(certificate_key) - - ### to renew a certificate: - with open('account_key.key', 'r') as account_key_file: - account_key = account_key_file.read() - - client = sewer.Client( - domain_name='example.com', provider=provider, - account_key=account_key - ) - certificate = client.renew() - certificate_key = client.certificate_key - - todo: handle more exceptions + refer to docs/sewer-as-a-library for usage, etc. """ def __init__( @@ -76,42 +41,6 @@ def __init__( ACME_VERIFY=True, LOG_LEVEL="INFO", ): - """ - :param domain_name: (required) [string] - the name that you want to acquire/renew certificate for. wildcards are allowed. - :param dns_class: (required) [class] - (DEPRECATED) a subclass of BaseDns which will be called to create/delete DNS TXT records. - do not pass this parameter if also passing provider. - :param provider: (required) [class] - a subclass of ProviderBase which will be called to create/delete auth records. - do not pass this parameter if also passing dns_class - :param domain_alt_names: (optional) [list] - list of alternative names that you want to be bundled into the same certificate as domain_name. - :param contact_email: (optional) [string] - a contact email address - :param account_key: (optional) [string] - a string whose contents is an ssl certificate that identifies your account on the acme server. - if you do not provide one, this client will issue a new certificate else will renew. - :param certificate_key: (optional) [string] - a string whose contents is a private key that will be incorporated into your new certificate. - if you do not provide one, this client will issue a new certificate else will renew. - :param bits: (optional) [integer] - number of bits that will be used to create your certificates' private key. - :param digest: (optional) [string] - the ssl digest type to be used in signing the certificate signing request(csr) - :param ACME_REQUEST_TIMEOUT: (optional) [integer] - the max time that the client will wait for a network call to complete. - :param ACME_AUTH_STATUS_WAIT_PERIOD: (optional) [integer] - the interval between two consecutive client polls on the acme server to check on authorization status - :param ACME_AUTH_STATUS_MAX_CHECKS: (optional) [integer] - the max number of times the client will poll the acme server to check on authorization status - :param ACME_DIRECTORY_URL: (optional) [string] - the url of the acme servers' directory endpoint - :param ACME_VERIFY: (optional) [bool] - suppress verification of SSL cert when set to False (for pebble); hint: -Wignore - :param LOG_LEVEL: (optional) [string] - the level to output log messages at. one of; 'DEBUG', 'INFO', 'WARNING', 'ERROR' or 'CRITICAL' - """ if not isinstance(domain_alt_names, (type(None), list)): raise ValueError( @@ -150,6 +79,7 @@ def __init__( "passed both the DEPRECATED 'dns_class' parameter as well as 'provider'." ) + # setup Client's global variables self.domain_name = domain_name self.provider = provider if provider is not None else dns_class if not domain_alt_names: @@ -205,7 +135,7 @@ def __init__( self.logger.info( "intialise_success, sewer_version={0}, domain_names={1}, acme_server={2}".format( - sewer_version.__version__, + sewer_about("version"), self.all_domain_names, self.ACME_DIRECTORY_URL[:20] + "...", ) @@ -277,8 +207,8 @@ def get_user_agent(): requests_version=requests.__version__, system=platform.system(), machine=platform.machine(), - sewer_version=sewer_version.__version__, - sewer_url=sewer_version.__url__, + sewer_version=sewer_about("version"), + sewer_url=sewer_about("url"), ) def get_acme_endpoints(self): diff --git a/sewer/dns_providers/tests/test_unbound_ssh.py b/sewer/dns_providers/tests/test_unbound_ssh.py new file mode 100644 index 00000000..8637cf79 --- /dev/null +++ b/sewer/dns_providers/tests/test_unbound_ssh.py @@ -0,0 +1,75 @@ +import logging +import unittest +from unittest import mock, TestCase + + +from .. import unbound_ssh + + +####### Mocks and other helpers ####### + + +class response: + "web request body content and or JSON nominally decoded from body" + + def __init__(self, *, content_val="", json_val=None): + self.content = content_val + self._json = json_val + + def json(self): + if self._json is None: + raise ValueError("No json here") + return self._json + + +class MockObj: + def __init__(self, **kwargs) -> None: + self.__dict__.update(kwargs) + + +def patch_subprocess_run(returncode, **kwargs): + return mock.patch("subprocess.run", return_value = MockObj(returncode = returncode, **kwargs)) + + +####### TESTS ####### + + +class TestLib(unittest.TestCase): + + # __init__ requires & accepts args, fails on missing + + def test01_init_requires_ssh_des(self): + with self.assertRaises(TypeError): + unbound_ssh.UnboundSsh() # pylint: disable=E1125 + + def test02_init_okay(self): + self.assertTrue(unbound_ssh.UnboundSsh(ssh_des="nobody@nowhere.man")) + + def test03_init_with_alias_okay(self): + self.assertTrue(unbound_ssh.UnboundSsh(ssh_des="nobody@nowhere.man", alias="example.com")) + + # local function unbound_command rejects invalid command + + def test13_unbound_command_bad_cmd_fails(self): + with self.assertRaises(ValueError): + unbound_ssh.unbound_command("bad", "fqdn", "acme_challenge") + + # end to end tests (up to calling out to ssh, of course) + + def test21_create_delete_dns_record_okay(self): + "test create and delete with the ssh callout mocked" + + provider = unbound_ssh.UnboundSsh(ssh_des="nobody@nowhere.man") + with patch_subprocess_run(0) as sub_run_mock: + provider.create_dns_record("example.com", "a1b2c3d4e5f6g7h8i9j0") + provider.delete_dns_record("example.com", "a1b2c3d4e5f6g7h8i9j0") + self.assertTrue(sub_run_mock.call_count == 2) + + def test22_create_dns_record_fail(self): + "only runs through create since the fail point is in the method both call" + + provider = unbound_ssh.UnboundSsh(ssh_des="nobody@nowhere.man") + with patch_subprocess_run(42, args=None) as sub_run_mock: + with self.assertRaises(RuntimeError): + provider.create_dns_record("example.com", "a1b2c3d4e5f6g7h8i9j0") + self.assertTrue(sub_run_mock.call_count == 1) diff --git a/sewer/lib.py b/sewer/lib.py index eb9b8aa6..dbb8286b 100644 --- a/sewer/lib.py +++ b/sewer/lib.py @@ -1,11 +1,11 @@ -import base64, logging +import base64, codecs, json, logging, os from hashlib import sha256 from typing import Any, Union LoggerType = logging.Logger -### FIX ME ### can be more specific about response's type... somehow +### FIX ME ### can be more specific about response arg's type... somehow def log_response(response: Any) -> str: @@ -47,3 +47,20 @@ def dns_challenge(key_auth: str) -> str: "return the ACME challenge response for a DNS TXT record" return safe_base64(sha256(key_auth.encode("utf8")).digest()) + + +_sewer_about = None + + +def sewer_about(name: str) -> str: + """ + returns the named attribute from lazily-loaded sewer.json (replaces __version__.py) + """ + + global _sewer_about + + if _sewer_about is None: + here = os.path.abspath(os.path.dirname(__file__)) + with codecs.open(os.path.join(here, "sewer.json"), "r", encoding="utf8") as f: + _sewer_about = json.load(f) + return _sewer_about[name] diff --git a/sewer/sewer.json b/sewer/sewer.json new file mode 100644 index 00000000..6aa3ba43 --- /dev/null +++ b/sewer/sewer.json @@ -0,0 +1,9 @@ +{ + "title": "sewer", + "description": "Sewer is a programmatic Lets Encrypt(ACME) client", + "url": "https://github.com/komuw/sewer", + "version": "0.8.3-beta", + "author": "komuW", + "author_email": "komuw05@gmail.com", + "license": "MIT" +} diff --git a/sewer/tests/test_catalog.py b/sewer/tests/test_catalog.py new file mode 100644 index 00000000..04a177d0 --- /dev/null +++ b/sewer/tests/test_catalog.py @@ -0,0 +1,26 @@ +import logging +import unittest + +from .. import auth, catalog + + +class TestLib(unittest.TestCase): + def test01_ProviderCatalog_create(self): + cat = catalog.ProviderCatalog() + self.assertIsInstance(cat, catalog.ProviderCatalog) + + def test02_catalog_get_item_list_okay(self): + cat = catalog.ProviderCatalog() + self.assertIsInstance(cat.get_item_list(), list) + + def test03_catalog_get_descriptor_okay(self): + cat = catalog.ProviderCatalog() + self.assertIsInstance(cat.get_descriptor("unbound_ssh"), catalog.ProviderDescriptor) + + def test04_catalog_get_provider_okay(self): + cat = catalog.ProviderCatalog() + provider = cat.get_provider("unbound_ssh")(ssh_des="noone@nowhere") + self.assertIsInstance(provider, auth.ProviderBase) + + def test_05_catalog__str__okay(self): + self.assertTrue(str(catalog.ProviderCatalog())) diff --git a/sewer/tests/test_lib.py b/sewer/tests/test_lib.py index 7470dbbd..3d895267 100644 --- a/sewer/tests/test_lib.py +++ b/sewer/tests/test_lib.py @@ -35,3 +35,7 @@ def test21_safe_base64_str_or_bytes_okay(self): def test31_dns_challenge_okay(self): res = lib.dns_challenge("a most spurious and unlikely key auth string") self.assertEqual(res, "lNNwvD6ceN7n6Iugd3m3k6HQD8Wk6ytGvKkwhHAV_Hw") + + def test41_sewer_about_okay(self): + res = lib.sewer_about("license") + self.assertEqual(res, "MIT")