From 4eff8cbe5e9f4f4d5f5ca7a77eb53755ad3c9b66 Mon Sep 17 00:00:00 2001 From: Jonathan Renon Date: Mon, 15 Jan 2024 16:27:01 +0100 Subject: [PATCH] Store the ansible-vault password in the system keyring by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We used to write the vault password to vault/vault_pass.txt, and we'll continue to work with existing clusters where that is the case. For new clusters, we'll try to use the keyring module to store the password into any available system keyring (e.g., gnome-keyring). Use `tpaexec configure … --keyring-backend ` to select a backend, but the only available choices are "system" (the default) and "legacy" (which is to use vault/vault_pass.txt). vault/vault_pass.txt method stores credentials inside the cluster directory and therefore there is no chance of conflict when multiple clusters using the same name are provisioned on a single tpa host, it's a challenge though when using a shared service. The change introduces an additional configuration setting named `vault_name` for the config.yml to go with the `keyring_backend`. For new clusters configured using `tpaexec configure` command, `vault_name` will be set to a UUID to make sure the combination of `cluster_name` and `vault_name` gives us a unique combination when `system` (default) is used as the keyring_backend. `vault_name` can be set to any arbitrary value to make sure the combination of `cluster_name` and `vault_name` is unique. It does not have to comply with UUID format. Also update tpaexec-configure.md to document `keyring_backend` and `vault_name` settings. Adds `show-vault` command; it displays the vault password for both `legacy` and `system` backends. With contributions from Abhijit and Haroon. References: TPA-85 --- .gitignore | 2 + architectures/lib/commands/show-password | 10 +- architectures/lib/commands/show-vault | 1 + architectures/lib/delete-vault | 26 +++ architectures/lib/generate-vault | 32 ++++ architectures/lib/use-vault | 32 ++++ bin/tpaexec | 20 +- docs/src/tpaexec-configure.md | 30 ++- lib/filter_plugins/filters.py | 2 +- lib/tpaexec/architecture.py | 232 +++++++++++++++-------- lib/tpaexec/password.py | 159 ++++++++++++++++ platforms/common/deprovision.yml | 13 ++ platforms/common/inventory/write.yml | 3 +- platforms/common/provision.yml | 28 ++- requirements.in | 1 + requirements.txt | 30 +++ roles/secret/tasks/main.yml | 16 +- roles/secret/vars/main.yml | 2 +- 18 files changed, 534 insertions(+), 105 deletions(-) create mode 120000 architectures/lib/commands/show-vault create mode 100755 architectures/lib/delete-vault create mode 100755 architectures/lib/generate-vault create mode 100755 architectures/lib/use-vault diff --git a/.gitignore b/.gitignore index 58108e066..969d90887 100644 --- a/.gitignore +++ b/.gitignore @@ -232,3 +232,5 @@ workflow/ coverage-reports/ lib/tests/config/* +requirements-dev.in +requirements-dev.txt diff --git a/architectures/lib/commands/show-password b/architectures/lib/commands/show-password index 20b133375..136b16f5d 100755 --- a/architectures/lib/commands/show-password +++ b/architectures/lib/commands/show-password @@ -7,11 +7,12 @@ import sys from posixpath import basename, join from ansible.cli import CLI +from ansible.errors import AnsibleFileNotFound from ansible.parsing.dataloader import DataLoader -from ansible import constants as C - from tpaexec.exceptions import PasswordReadError +from ansible import constants as C + prog = "show-password" p = argparse.ArgumentParser( prog=prog, @@ -52,7 +53,7 @@ try: ) except: raise PasswordReadError( - "vault_password_file: {} not found".format(args.get("vault_password_file")) + f"vault_password_file: {args.get('vault_password_file')} not found" ) try: @@ -60,5 +61,4 @@ try: print(data[password_filename]) except: raise PasswordReadError( - "password not found for {} at {}".format(args.get("user"), password_file) - ) + f"password not found for {args.get('user')} at {password_file}") diff --git a/architectures/lib/commands/show-vault b/architectures/lib/commands/show-vault new file mode 120000 index 000000000..6ee7ecdd1 --- /dev/null +++ b/architectures/lib/commands/show-vault @@ -0,0 +1 @@ +../use-vault \ No newline at end of file diff --git a/architectures/lib/delete-vault b/architectures/lib/delete-vault new file mode 100755 index 000000000..29c233a51 --- /dev/null +++ b/architectures/lib/delete-vault @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# © Copyright EnterpriseDB UK Limited 2015-2023 - All rights reserved. + +import sys +from tpaexec.password import delete_password, exists + + +def main( + cluster_dir, keyring_backend="system", password_name="vault_pass" +): + """ + delete vault password in chosen keyring backend + exit code 0 if a password was deleted, otherwise exit code 2. + """ + if exists(cluster_dir, password_name, keyring_backend): + delete_password(cluster_dir, password_name, keyring_backend) + sys.exit(0) + else: + sys.exit(2) + + +if __name__ == "__main__": + from sys import argv + + main(*argv[1:]) diff --git a/architectures/lib/generate-vault b/architectures/lib/generate-vault new file mode 100755 index 000000000..d5e5ebfb7 --- /dev/null +++ b/architectures/lib/generate-vault @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# © Copyright EnterpriseDB UK Limited 2015-2023 - All rights reserved. + +import sys +from os import path +from tpaexec.password import store_password, exists, generate_password + + +def main( + cluster_dir, keyring_backend=None, password_name="vault_pass", +): + """ + generate and store vault password in chosen keyring backend + exit code 0 if new password was generated else exit code 2 if password already existed. + """ + if not exists(path.basename(path.abspath(cluster_dir)), password_name, keyring_backend): + store_password( + cluster_dir, + password_name, + generate_password(), + keyring_backend, + ) + sys.exit(0) + else: + sys.exit(2) + + +if __name__ == "__main__": + from sys import argv + + main(*argv[1:]) diff --git a/architectures/lib/use-vault b/architectures/lib/use-vault new file mode 100755 index 000000000..e9cc4893d --- /dev/null +++ b/architectures/lib/use-vault @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# © Copyright EnterpriseDB UK Limited 2015-2023 - All rights reserved. +from os import getenv, getcwd +from os import path + +import yaml +from tpaexec.password import show_password + + +def main(): + """ + retrieve vault password from chosen keyring backend + """ + cluster_dir = getcwd() + try: + with open(path.join(cluster_dir, "config.yml")) as config_file: + config = yaml.safe_load(config_file) + keyring_backend = config.get( + "keyring_backend", getenv("TPA_KEYRING_BACKEND", None) + ) + password_name = config.get( + "vault_name", "vault_pass") + except IOError: + print(f"could not load {path.join(cluster_dir, 'config.yml')}") + raise + + show_password(cluster_dir, password_name, keyring_backend) + + +if __name__ == "__main__": + main() diff --git a/bin/tpaexec b/bin/tpaexec index d4227e600..d23ce6de3 100755 --- a/bin/tpaexec +++ b/bin/tpaexec @@ -352,8 +352,13 @@ provision() { } cmd() { + # first ensure that vault_pass is defined for the cluster + # otherwise don't add it to the args + if "${TPA_DIR}/architectures/lib/use-vault" >> /dev/null 2>&1; then + args=--vault-password-file="${TPA_DIR}/architectures/lib/use-vault" + fi $ansible \ - -i inventory --vault-password-file vault/vault_pass.txt \ + -i inventory "$args"\ -e cluster_dir="$(pwd)" "$@" } @@ -369,8 +374,8 @@ playbook() { args+=(-i inventory) fi - if [ -f vault/vault_pass.txt ]; then - args+=(--vault-password-file vault/vault_pass.txt) + if "${TPA_DIR}/architectures/lib/use-vault" >> /dev/null 2>&1; then + args+=(--vault-password-file "${TPA_DIR}/architectures/lib/use-vault") fi "$ansible"-playbook "${args[@]}" "$@" @@ -422,7 +427,7 @@ try_as_playbook_or_script() { if [[ -x $exec ]]; then case $command in store-password|show-password) - _with_ansible_env "$exec" "$@" "--vault_password_file=vault/vault_pass.txt" + _with_ansible_env "$exec" "$@" "--vault_password_file=${TPA_DIR}/architectures/lib/use-vault" return $? ;; *) @@ -611,6 +616,9 @@ Available help topics: store-password Commands to manage passwords for cluster users (e.g., postgres_password, pgbouncer_password) + show-vault + Command to show the vault password of the cluster + This is used to encrypt other passwords. Miscellaneous @@ -1156,14 +1164,14 @@ case "$command" in cmd|ping|provision|deploy|upgrade|playbook|deprovision) time $command "$@" real_exit_status=$? - playbook ${TPA_DIR}/architectures/lib/commands/check-repositories.yml + playbook "${TPA_DIR}/architectures/lib/commands/check-repositories.yml" exit $real_exit_status ;; *) try_as_playbook_or_script "$@" real_exit_status=$? - playbook ${TPA_DIR}/architectures/lib/commands/check-repositories.yml + playbook "${TPA_DIR}/architectures/lib/commands/check-repositories.yml" exit $real_exit_status ;; esac diff --git a/docs/src/tpaexec-configure.md b/docs/src/tpaexec-configure.md index cc33b7bb3..86a457e51 100644 --- a/docs/src/tpaexec-configure.md +++ b/docs/src/tpaexec-configure.md @@ -395,7 +395,7 @@ Use the `--use-ansible-tower` and `--tower-git-repository` options to create a cluster adapted for deployment with Ansible Tower. See [Ansible Tower](tower.md) for details. -## git repository +## Git repository By default, a git repository is created with an initial branch named after the cluster, and a single commit is made, with the configure @@ -406,6 +406,34 @@ option. (Note that in an Ansible Tower cluster, a git repository is required and will be created later by `tpaexec provision` if it does not already exist.) +## Keyring backend for vault password + +TPA generates a cluster specific ansible vault password. +This password is used to encrypt other sensitive variables generated +for the cluster, postgres user password, barman user password and so on. + +Keyring backend `system` will leverage the best keyring backend on your system +from the list of supported backend by python keyring module including +gnome-keyring and secret-tool. + +Default is to store the vault password using `system` keyring for new cluster. +removing `keyring_backend: system` in config.yml file **before** any `provision` +will revert previous default to store vault password in plaintext file. + +Using `keyring_backend: system` also generates a `vault_name` entry in config.yml +used to store the vault password unique storage name. TPA generate an UUID by +default but there is no naming scheme requirements. + +Note: When using `keyring_backend: system` and the same base config.yml file +for multiple clusters with same `cluster_name`, by copying the config file to +a different location, ensure the value pair (`vault_name`, `cluster_name`) +is unique for each cluster copy. + +Note: When using `keyring_backend: system` and moving an already provisioned +cluster folder to a different tpa host, ensure that you export the associated +vault password on the new machine's system keyring. vault password can be +displayed via `tpaexec show-vault `. + ## Examples Let's see what happens when we run the following command: diff --git a/lib/filter_plugins/filters.py b/lib/filter_plugins/filters.py index c19080316..5f24e5176 100644 --- a/lib/filter_plugins/filters.py +++ b/lib/filter_plugins/filters.py @@ -265,7 +265,7 @@ def cmdline(playbook_dir): and args[5] == "-i" and args[6] == "inventory" and args[7] == "--vault-password-file" - and args[8] == "vault/vault_pass.txt" + and args[8].endswith("use-vault") ): tpaexec = "tpaexec" diff --git a/lib/tpaexec/architecture.py b/lib/tpaexec/architecture.py index d82027367..17effa8e5 100644 --- a/lib/tpaexec/architecture.py +++ b/lib/tpaexec/architecture.py @@ -4,6 +4,7 @@ import sys import os import io +import uuid import subprocess import argparse import shutil @@ -22,6 +23,8 @@ from .net import Network, DEFAULT_SUBNET_PREFIX_LENGTH, DEFAULT_NETWORK_CIDR from .platforms import Platform +KEYRING_SUPPORTED_BACKENDS = ["system", "legacy"] + class Architecture(object): """ @@ -214,11 +217,13 @@ def add_options(self, p): g.add_argument( "--enable-pem", action="store_true", + default=argparse.SUPPRESS, help="add a PEM monitoring server to the cluster", ) g.add_argument( "--enable-pg-backup-api", action="store_true", + default=argparse.SUPPRESS, help="install pg-backup-api on barman server and register to pem server if enabled", ) g.add_argument("--extra-packages", nargs="+", metavar="NAME") @@ -338,7 +343,7 @@ def __call__(self, parser, namespace, arg, option_string=None): "--redwood", action="store_true", dest="epas_redwood_compat", - default=None, + default=argparse.SUPPRESS, help="enable Oracle compatibility features for EPAS", ) g_redwood.add_argument( @@ -346,7 +351,7 @@ def __call__(self, parser, namespace, arg, option_string=None): "--no-redwood-compat", action="store_false", dest="epas_redwood_compat", - default=None, + default=argparse.SUPPRESS, help="disable Oracle compatibility features for EPAS", ) @@ -355,12 +360,14 @@ def __call__(self, parser, namespace, arg, option_string=None): g_local.add_argument( "--enable-local-repo", action="store_true", + default=argparse.SUPPRESS, help="make packages available to instances from cluster_dir/local-repo", ) g_local.add_argument( "--use-local-repo-only", action="store_true", help="make packages available to instances from cluster_dir/local-repo, and disable all other repositories", + default=argparse.SUPPRESS, ) g = p.add_argument_group("volume sizes in GB") @@ -394,7 +401,13 @@ def __call__(self, parser, namespace, arg, option_string=None): g = p.add_argument_group("locations") g.add_argument("--location-names", metavar="LOCATION", nargs="+") - g = p.add_argument_group("host configuration") + g = p.add_argument_group("password management") + g.add_argument( + "--keyring-backend", + help="backend used to store cluster's vault password", + choices=KEYRING_SUPPORTED_BACKENDS, + default=argparse.SUPPRESS, + ) def add_architecture_options(self, p, g): """ @@ -459,7 +472,33 @@ def validate_arguments(self, args): # By now, both postgres_flavour and postgres_version must be set, # whether they were specified separately or through a shortcut like # `--postgresql 14`. + self._validate_flavour_version(args) + + # Validate arguments to --2Q-repositories + self._validate_2q_repositories(args) + + # Validate arguments to --install-from-source + self._validate_from_source(args) + + # --use-local-repo-only implies --enable-local-repo + if self.args.get("use_local_repo_only"): + self.args["enable_local_repo"] = True + # use either command line or environment variable + # to determine keyring backend, default to system. + if not self.args.get("keyring_backend"): + self.args["keyring_backend"] = os.environ.get( + "TPA_KEYRING_BACKEND", "system" + ) + + self.platform.validate_arguments(args) + + def _validate_flavour_version(self, args): + """Verify postgres flavour, version and related arguments. + By now, both postgres_flavour and postgres_version must be set, + whether they were specified separately or through a shortcut like + `--postgresql 14`. + """ flavour = args.get("postgres_flavour") version = args.get("postgres_version") @@ -509,32 +548,39 @@ def validate_arguments(self, args): "PEM is not compatible with the BDR-Always-ON architecture and EDB Postgres Extended" ) - # Validate arguments to --2Q-repositories + def _validate_2q_repositories(self, args): + """Validate arguments to --2Q-repositories""" repos = args.get("tpa_2q_repositories") or [] for r in repos: errors = [] parts = r.split("/") if len(parts) == 3: - (source, name, maturity) = parts - if source not in ["ci-spool", "products", "dl"]: - errors.append( - "unknown source '%s' (try 'dl', 'products', or 'ci-spool')" - % source - ) - if name not in self.product_repositories(): - errors.append("unknown product name '%s'" % name) - if maturity not in ["snapshot", "testing", "release"]: - errors.append( - "unknown maturity '%s' (try 'release', 'testing', or 'snapshot')" - % maturity - ) + errors = self._2q_repo_exists(parts, errors) else: errors.append("invalid name (expected source/product/maturity)") if errors: raise ArchitectureError(*(f"repository '{r}' has {e}" for e in errors)) - # Validate arguments to --install-from-source + def _2q_repo_exists(self, parts, errors): + """Ensure each part of a 2q repo is a valid input""" + (source, name, maturity) = parts + if source not in ["ci-spool", "products", "dl"]: + errors.append( + "unknown source '%s' (try 'dl', 'products', or 'ci-spool')" % source + ) + if name not in self.product_repositories(): + errors.append("unknown product name '%s'" % name) + if maturity not in ["snapshot", "testing", "release"]: + errors.append( + "unknown maturity '%s' (try 'release', 'testing', or 'snapshot')" + % maturity + ) + return errors + + def _validate_from_source(self, args): + """Validate arguments to --install-from-source""" + errors = [] source_names = [] sources = args.get("install_from_source") or [] @@ -560,12 +606,6 @@ def validate_arguments(self, args): if errors: raise ArchitectureError(*(f"--install-from-source {e}" for e in errors)) - # --use-local-repo-only implies --enable-local-repo - if self.args.get("use_local_repo_only"): - self.args["enable_local_repo"] = True - - self.platform.validate_arguments(args) - def process_arguments(self, args): """ Augment arguments from the command-line with enough additional @@ -629,6 +669,8 @@ def process_arguments(self, args): self.platform.update_cluster_tags(cluster_tags, args) args["cluster_tags"] = cluster_tags + self._init_top_level_settings() + cluster_vars = args.get("cluster_vars", {}) self._init_cluster_vars(cluster_vars) self.update_cluster_vars(cluster_vars) @@ -849,6 +891,37 @@ def update_cluster_tags(self, cluster_tags): """ pass + def _init_top_level_settings(self): + """Add top level settings applicable accross all architectures""" + self._add_tower_settings() + + self._add_keyring_settings() + + def _add_tower_settings(self): + """Add top level settings for Tower""" + if self.args.get("tower_api_url"): + top = self.args.get("top_level_settings") or {} + top.update({"use_ssh_agent": "true"}) + + tower = self.args.get("tower_settings") or {} + tower.update({"api_url": self.args["tower_api_url"]}) + + self.args["tower_settings"] = tower + self.args["top_level_settings"] = top + + if self.args.get("tower_git_repository"): + tower = self.args.get("tower_settings") or {} + tower.update({"git_repository": self.args["tower_git_repository"]}) + self.args["tower_settings"] = tower + + def _add_keyring_settings(self): + if self.args.get("keyring_backend"): + top = self.args.get("top_level_settings") or {} + top.update({"keyring_backend": self.args.get("keyring_backend")}) + if self.args.get("keyring_backend") == "system": + top.update({"vault_name": str(uuid.uuid4())}) + self.args["top_level_settings"] = top + def _init_cluster_vars(self, cluster_vars): """ Makes changes to cluster_vars applicable across architectures @@ -860,14 +933,25 @@ def _init_cluster_vars(self, cluster_vars): "preferred_python_version", preferred_python_version ) + self._add_cluster_vars_args(cluster_vars) + + if self.args.get("postgres_flavour") == "epas": + k = "epas_redwood_compat" + cluster_vars[k] = cluster_vars.get(k, self.args.get(k)) + + self._add_extra_packages(cluster_vars) + + self._add_source_install(cluster_vars) + + def _add_cluster_vars_args(self, cluster_vars): + """Add args that belongs to cluster_vars without any change or logic""" for k in self.cluster_vars_args(): val = self.args.get(k) if val is not None: cluster_vars[k] = cluster_vars.get(k, val) - if self.args.get("postgres_flavour") == "epas": - k = "epas_redwood_compat" - cluster_vars[k] = cluster_vars.get(k, self.args.get(k)) + def _add_extra_packages(self, cluster_vars): + """Add extra packages lists to cluster_vars""" package_option_vars = { "extra_packages": "packages", @@ -882,43 +966,12 @@ def _init_cluster_vars(self, cluster_vars): val = {"common": packages} cluster_vars[var] = cluster_vars.get(var, val) + def _add_source_install(self, cluster_vars): + """Add --install-from-source entries into cluster_vars""" sources = self.args.get("install_from_source") or [] - local_sources = self.args.get("local_sources") or {} - installable_sources = self.installable_sources() install_from_source = [] for name in sources: - ref = None - if ":" in name: - (name, ref) = name.split(":", 1) - name = name.lower() - entry = installable_sources[name] - - if ref and name in local_sources: - raise ArchitectureError( - f"--install-from-source can't guarantee {name}:{ref} while using local source" - f" directory {local_sources[name].split(':')[0]}" - ) - - if name in ["postgres", "2ndqpostgres"]: - if ref: - entry.update({"postgres_git_ref": ref}) - cluster_vars.update(entry) - elif name == "barman": - if ref: - entry.update({"barman_git_ref": ref}) - cluster_vars.update(entry) - elif name == "pg-backup-api": - if ref: - entry.update({"pg_backup_api_git_ref": ref}) - cluster_vars.update(entry) - elif name in ["patroni", "patroni-edb"]: - if ref: - entry.update({"patroni_git_ref": ref}) - cluster_vars.update(entry) - else: - if ref: - entry.update({"git_repository_ref": ref}) - install_from_source.append(entry) + self._update_source_refs(name, cluster_vars, install_from_source) if install_from_source: cluster_vars["install_from_source"] = cluster_vars.get( @@ -930,23 +983,45 @@ def _init_cluster_vars(self, cluster_vars): top.update({"forward_ssh_agent": "yes"}) self.args["top_level_settings"] = top - if self.args.get("use_local_repo_only"): - cluster_vars["use_local_repo_only"] = True - - if self.args.get("tower_api_url"): - top = self.args.get("top_level_settings") or {} - top.update({"use_ssh_agent": "true"}) - - tower = self.args.get("tower_settings") or {} - tower.update({"api_url": self.args["tower_api_url"]}) - - self.args["tower_settings"] = tower - self.args["top_level_settings"] = top - - if self.args.get("tower_git_repository"): - tower = self.args.get("tower_settings") or {} - tower.update({"git_repository": self.args["tower_git_repository"]}) - self.args["tower_settings"] = tower + def _update_source_refs(self, name, cluster_vars, install_from_source): + installable_sources = self.installable_sources() + ref = None + if ":" in name: + (name, ref) = name.split(":", 1) + name = name.lower() + entry = installable_sources[name] + + self._check_local_sources(name, ref) + + if name in ["postgres", "2ndqpostgres", "epas"]: + if ref: + entry.update({"postgres_git_ref": ref}) + cluster_vars.update(entry) + elif name == "barman": + if ref: + entry.update({"barman_git_ref": ref}) + cluster_vars.update(entry) + elif name == "pg-backup-api": + if ref: + entry.update({"pg_backup_api_git_ref": ref}) + cluster_vars.update(entry) + elif name in ["patroni", "patroni-edb"]: + if ref: + entry.update({"patroni_git_ref": ref}) + cluster_vars.update(entry) + else: + if ref: + entry.update({"git_repository_ref": ref}) + install_from_source.append(entry) + + def _check_local_sources(self, name, ref): + """Check that we don't fix a ref when using local source""" + local_sources = self.args.get("local_sources") or {} + if ref and name in local_sources: + raise ArchitectureError( + f"--install-from-source can't guarantee {name}:{ref} while using local source" + f" directory {local_sources[name].split(':')[0]}" + ) def postgres_eol_repos(self, cluster_vars): if ( @@ -1159,6 +1234,7 @@ def cluster_vars_args(self): "postgres_version", "tpa_2q_repositories", "use_volatile_subscriptions", + "use_local_repo_only", "failover_manager", "enable_pg_backup_api", ] + ["%s_package_version" % x for x in self.versionable_packages()] diff --git a/lib/tpaexec/password.py b/lib/tpaexec/password.py index dab5a48bb..873022d50 100644 --- a/lib/tpaexec/password.py +++ b/lib/tpaexec/password.py @@ -2,8 +2,17 @@ # -*- coding: utf-8 -*- # © Copyright EnterpriseDB UK Limited 2015-2024 - All rights reserved. +import os +import sys +import keyring +from tpaexec.architecture import KEYRING_SUPPORTED_BACKENDS from passlib import pwd +KEYRING_PREFIX = "TPA_" +VAULT_PASS_RELATIVE_PATH = "vault/vault_pass.txt" +NO_KEYRING_ERROR_MSG = """Could not find compatible keyring backend, +ensure that you have a compatible backend for python keyring module +or use keyring_backend: legacy in config.yml""" def generate_password(): """ @@ -26,3 +35,153 @@ def generate_password(): # details about the password generation. return pwd.genword(entropy="secure", length=32, charset="ascii_72") + + +def store_password(cluster_dir, password_name, password, keyring_backend): + """Leverage the chosen keyring backend to store a password for the cluster + + Args: + cluster_dir (string): path of the cluster the password belongs to. + password_name (string): name of the password entry to store + password (string): password to store + keyring_backend (string): name of keychain backend used to store the password + """ + cluster_name = os.path.basename(os.path.abspath(cluster_dir)) + _initialize_keyring( + cluster_dir=cluster_dir, + keyring_backend=keyring_backend, + ) + + try: + keyring.set_password( + KEYRING_PREFIX + cluster_name, username=password_name, password=password + ) + except keyring.errors.PasswordSetError: + print( + f"Failed to store password: {password_name} in system {KEYRING_PREFIX + cluster_name}" + ) + exit(1) + except keyring.errors.NoKeyringError: + print(NO_KEYRING_ERROR_MSG) + exit(1) + + +def show_password(cluster_dir, password_name, keyring_backend): + """Display a password stored into a keyring backend + + Args: + cluster_dir (string): path of the cluster the password belongs to. + password_name (string): name of the password entry to store + keyring_backend (string): name of keychain backend used to store the password + Returns: + string: password value or None + """ + _initialize_keyring( + cluster_dir=cluster_dir, + keyring_backend=keyring_backend, + ) + cluster_name = os.path.basename(os.path.abspath(cluster_dir)) + + # legacy plain text file. + if keyring_backend is None or keyring_backend == "legacy": + try: + with open("/".join([cluster_dir, VAULT_PASS_RELATIVE_PATH])) as vault_file: + return print(vault_file.read().strip("\n")) + except IOError: + print( + f"Could not open the vault_file: {'/'.join([cluster_dir, VAULT_PASS_RELATIVE_PATH])}" + ) + exit(3) + + elif keyring_backend in KEYRING_SUPPORTED_BACKENDS: + try: + password = keyring.get_password(KEYRING_PREFIX + cluster_name, password_name) + except keyring.errors.NoKeyringError: + print(NO_KEYRING_ERROR_MSG) + exit(1) + if password is None: + sys.exit( + f"""Could not find vault password in system keyring, please use --ask-vault-pass +or ensure entry: +service: {KEYRING_PREFIX + cluster_name} +username: {password_name} +exists with the correct vault password by running: +`tpaexec provision {cluster_dir}` +or +`keyring set {KEYRING_PREFIX + cluster_name} {password_name}`""" + ) + return print(password) + else: + sys.exit(1) + + +def delete_password(cluster_dir, password_name, keyring_backend): + """delete password from keyring backend + + Args: + cluster_dir (string): path of the cluster the password belongs to. + keyring_backend (string): name of keychain backend used to store the password + password_name (string): name of the password entry to delete + """ + _initialize_keyring( + cluster_dir=cluster_dir, + keyring_backend=keyring_backend, + ) + + cluster_name = os.path.basename(os.path.abspath(cluster_dir)) + if keyring_backend is None or keyring_backend == "legacy": + os.remove("/".join([cluster_dir, VAULT_PASS_RELATIVE_PATH])) + elif keyring_backend in KEYRING_SUPPORTED_BACKENDS: + keyring.delete_password(KEYRING_PREFIX + cluster_name, password_name) + + +def exists(cluster_dir, password_name, keyring_backend): + """Returns true if password already exists otherwise returns false + + Args: + cluster_dir (string): path of the cluster the password belongs to. + password_name (string): name of the password entry to store + keyring_backend (string): name of keychain backend used to store the password + + Returns: + _type_: _description_ + """ + _initialize_keyring( + cluster_dir=cluster_dir, + keyring_backend=keyring_backend, + ) + cluster_name = os.path.basename(os.path.abspath(cluster_dir)) + if ( + keyring_backend in KEYRING_SUPPORTED_BACKENDS + and keyring_backend != "legacy" + ): + try: + if keyring.get_password(KEYRING_PREFIX + cluster_name, password_name) is None: + return False + except keyring.errors.NoKeyringError: + return False + + elif ( + keyring_backend is None or keyring_backend == "legacy" + ) and not os.path.exists("/".join([cluster_dir, VAULT_PASS_RELATIVE_PATH])): + return False + return True + + +def _initialize_keyring(cluster_dir, keyring_backend): + """Prepare keyring module to use the selected backend + + Args: + cluster_dir (string): path of the cluster the password belongs to. + keyring_backend (string): name of keychain backend to initialize + """ + # will start with keyring only and we will probably need more python module to + # support more use case (hashicorp vault seems to have no good keyring + # implementation and could require hvac module instead) + + # python keyring backend module supported + # this will allow for other module to be used if needed. + # nothing to do but keeping the example for later use. + if keyring_backend in KEYRING_SUPPORTED_BACKENDS: + if keyring_backend == "system": + pass \ No newline at end of file diff --git a/platforms/common/deprovision.yml b/platforms/common/deprovision.yml index 4534f86b9..1fb29cdec 100644 --- a/platforms/common/deprovision.yml +++ b/platforms/common/deprovision.yml @@ -64,3 +64,16 @@ changed_when: > rmdir.rc == 99 and 'Directory not empty' not in rmdir.stderr failed_when: rmdir.rc not in [0, 99] + +- name: Delete vault password (system) + shell: > + "{{ tpa_dir }}"/architectures/lib/delete-vault + "{{ cluster_dir }}" + "{{ keyring_backend }}" + "{{ vault_name }}" + args: + executable: /bin/bash + register: rmvault + changed_when: rmvault.rc == 0 + failed_when: rmvault.rc not in [0, 2] + when: keyring_backend is defined and keyring_backend == "system" \ No newline at end of file diff --git a/platforms/common/inventory/write.yml b/platforms/common/inventory/write.yml index f96ce159b..855d2c555 100644 --- a/platforms/common/inventory/write.yml +++ b/platforms/common/inventory/write.yml @@ -41,6 +41,7 @@ 'cluster_tag': cluster_tag, 'cluster_name': cluster_name, 'tpa_version': tpa_version, + 'keyring_backend': keyring_backend|default(None) }) }}" - name: Add ssh_key_file to cluster_group_vars @@ -165,7 +166,7 @@ include_role: name=secret vars: secret_name: "{{ item }}" - lock_file: "{{ cluster_dir }}/vault/vault_pass.txt" + lock_file: "{{ tpa_dir }}/architectures/lib/use-vault" with_items: - postgres_password - barman_password diff --git a/platforms/common/provision.yml b/platforms/common/provision.yml index 1c859ec49..c61111484 100644 --- a/platforms/common/provision.yml +++ b/platforms/common/provision.yml @@ -54,16 +54,34 @@ - ecdsa tags: [common, ssh, hostkeys] -# We generate a random vault passphrase and store it in a text file. -# This obviously needs improvement—we should GPG-encrypt it with the -# public keys of everyone who needs access to the secret. +# We generate a random vault passphrase and store it in the system keyring +# if any available and supported by python keyring module. -- name: Generate the vault passphrase for this cluster +- name: Generate the vault passphrase using keyring backend {{ keyring_backend }} + command: > + "{{ tpa_dir }}"/architectures/lib/generate-vault + "{{ cluster_dir }}" + "{{ keyring_backend }}" + "{{ vault_name }}" + register: vault_gen + changed_when: vault_gen.rc == 0 + failed_when: vault_gen.rc not in [0,2] + when: > + keyring_backend is defined + and keyring_backend != "legacy" + tags: [common, vault] + +# When using `keyring_backend: legacy` or if not set we revert to +# storing vault password in a plain text file. + +- name: Generate the vault passphrase using plaintext shell: > "{{ tpa_dir }}"/architectures/lib/password > vault_pass.txt args: chdir: "{{ cluster_dir }}/vault" creates: vault_pass.txt executable: /bin/bash - delegate_to: localhost + when: > + keyring_backend is not defined + or keyring_backend == "legacy" tags: [common, vault] diff --git a/requirements.in b/requirements.in index 7a7a7c634..bae578d6c 100644 --- a/requirements.in +++ b/requirements.in @@ -14,3 +14,4 @@ certifi>=2023.7.22 docker passlib psutil +keyring \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 59893b398..3935e5f8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -208,6 +208,20 @@ idna==3.6 \ --hash=sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca \ --hash=sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f # via requests +importlib-metadata==7.0.1 \ + --hash=sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e \ + --hash=sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc + # via keyring +jaraco-classes==3.3.0 \ + --hash=sha256:10afa92b6743f25c0cf5f37c6bb6e18e2c5bb84a16527ccfc0040ea377e7aaeb \ + --hash=sha256:c063dd08e89217cee02c8d5e5ec560f2c8ce6cdc2fcdc2e68f7b2e5547ed3621 + # via keyring +jeepney==0.8.0 \ + --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \ + --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755 + # via + # keyring + # secretstorage jinja2==3.1.3 \ --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \ --hash=sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90 @@ -218,6 +232,10 @@ jmespath==1.0.1 \ # via # boto3 # botocore +keyring==24.3.0 \ + --hash=sha256:4446d35d636e6a10b8bce7caa66913dd9eca5fd222ca03a3d42c38608ac30836 \ + --hash=sha256:e730ecffd309658a08ee82535a3b5ec4b4c8669a9be11efb66249d8e0aeb9a25 + # via -r requirements.in markupsafe==2.1.3 \ --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ @@ -280,6 +298,10 @@ markupsafe==2.1.3 \ --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via jinja2 +more-itertools==10.2.0 \ + --hash=sha256:686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684 \ + --hash=sha256:8fccb480c43d3e99a00087634c06dd02b0d50fbf088b380de5a41a015ec239e1 + # via jaraco-classes netaddr==0.10.1 \ --hash=sha256:9822305b42ea1020d54fee322d43cee5622b044c07a1f0130b459bb467efcf88 \ --hash=sha256:f4da4222ca8c3f43c8e18a8263e5426c750a3a837fdfeccf74c68d0408eaa3bf @@ -380,6 +402,10 @@ s3transfer==0.10.0 \ --hash=sha256:3cdb40f5cfa6966e812209d0994f2a4709b561c88e90cf00c2696d2df4e56b2e \ --hash=sha256:d0c8bbf672d5eebbe4e57945e23b972d963f07d82f661cabf678a5c88831595b # via boto3 +secretstorage==3.3.3 \ + --hash=sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77 \ + --hash=sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99 + # via keyring six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -391,3 +417,7 @@ urllib3==1.26.18 \ # botocore # docker # requests +zipp==3.17.0 \ + --hash=sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31 \ + --hash=sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0 + # via importlib-metadata diff --git a/roles/secret/tasks/main.yml b/roles/secret/tasks/main.yml index 92c822e46..6646b2fc6 100644 --- a/roles/secret/tasks/main.yml +++ b/roles/secret/tasks/main.yml @@ -16,13 +16,14 @@ shell: | ( flock -n 9 || exit 99; - ansible-vault encrypt \ - --vault-password-file "{{ _vault_passfile }}" \ - --output "{{ _secret_file }}" \ - <<< '{{ _payload|to_nice_yaml(default_style='\"') }}' \ + ansible-vault encrypt_string \ + --vault-password-file {{ _vault_passfile }} \ + --name {{ secret_name }} \ + '{{ _payload }}' \ + > {{ _secret_file }} \ || exit $? exit 88 - ) 9<"{{ _lockfile }}" + ) 9<{{ _lockfile }} args: creates: "{{ _secret_file }}" executable: /bin/bash @@ -34,11 +35,12 @@ no_log: true become: no vars: - _vault_passfile: "{{ _vault_dir }}/vault_pass.txt" + _vault_passfile: "{{ _vault_dir }}/use-vault" _secret_file: "{{ '%s/inventory/group_vars/%s/secrets/%s.yml' % (cluster_dir, cluster_tag, secret_name) }}" - _payload: "{{ {secret_name: lookup('pipe', tpa_dir + '/architectures/lib/password')} }}" + _payload: "{{ lookup('pipe', tpa_dir + '/architectures/lib/password')|string|trim }}" _lockfile: "{{ lock_file | default(inventory_file) }}" delegate_to: localhost + run_once: true # We created group_vars/$cluster_tag/secrets/$secret_name.yml, which # Ansible will load automatically on subsequent runs; but we also need diff --git a/roles/secret/vars/main.yml b/roles/secret/vars/main.yml index 9a2f25184..e0ace2ab3 100644 --- a/roles/secret/vars/main.yml +++ b/roles/secret/vars/main.yml @@ -2,4 +2,4 @@ # © Copyright EnterpriseDB UK Limited 2015-2024 - All rights reserved. -_vault_dir: "{{ cluster_dir }}/vault" +_vault_dir: "{{ tpa_dir }}/architectures/lib/"