Store the ansible-vault password in the system keyring by default

We used to write the vault password to vault/vault_pass.txt, and we'll
continue to work with existing clusters where that is the case. For new
clusters, we'll try to use the keyring module to store the password into
any available system keyring (e.g., gnome-keyring).

Use `tpaexec configure … --keyring-backend <name>` to select a backend,
but the only available choices are "system" (the default) and "legacy"
(which is to use vault/vault_pass.txt).

vault/vault_pass.txt method stores credentials inside the cluster
directory and therefore there is no chance of conflict when multiple
clusters using the same name are provisioned on a single tpa host, it's
a challenge though when using a shared service. The change introduces an
additional configuration setting named `vault_name` for the config.yml
to go with the `keyring_backend`. For new clusters configured using
`tpaexec configure` command, `vault_name` will be set to a UUID to make
sure the combination of `cluster_name` and `vault_name` gives us a
unique combination when `system` (default) is used as the
keyring_backend. `vault_name` can be set to any arbitrary value to make
sure the combination of `cluster_name` and `vault_name` is unique. It
does not have to comply with UUID format.

Also update tpaexec-configure.md to document `keyring_backend` and
`vault_name` settings.

Adds `show-vault` command; it displays the vault password for both
`legacy` and `system` backends.

With contributions from Abhijit and Haroon.

References: TPA-85

.gitignore

@@ -232,3 +232,5 @@ workflow/
 
 coverage-reports/
 lib/tests/config/*
+requirements-dev.in
+requirements-dev.txt

architectures/lib/commands/show-password

@@ -7,11 +7,12 @@ import sys
 from posixpath import basename, join
 
 from ansible.cli import CLI
+from ansible.errors import AnsibleFileNotFound
 from ansible.parsing.dataloader import DataLoader
-from ansible import constants as C
-
 from tpaexec.exceptions import PasswordReadError
 
+from ansible import constants as C
+
 prog = "show-password"
 p = argparse.ArgumentParser(
     prog=prog,
@@ -52,13 +53,12 @@ try:
     )
 except:
     raise PasswordReadError(
-        "vault_password_file: {} not found".format(args.get("vault_password_file"))
+        f"vault_password_file: {args.get('vault_password_file')} not found"
     )
 
 try:
     data = loader.load_from_file(password_file)
     print(data[password_filename])
 except:
     raise PasswordReadError(
-        "password not found for {} at {}".format(args.get("user"), password_file)
-    )
+        f"password not found for {args.get('user')} at {password_file}")

architectures/lib/commands/show-vault

@@ -0,0 +1 @@
+../use-vault

architectures/lib/delete-vault

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# © Copyright EnterpriseDB UK Limited 2015-2023 - All rights reserved.
+
+import sys
+from tpaexec.password import delete_password, exists
+
+
+def main(
+    cluster_dir, keyring_backend="system", password_name="vault_pass"
+):
+    """
+    delete vault password in chosen keyring backend
+    exit code 0 if a password was deleted, otherwise exit code 2.
+    """
+    if exists(cluster_dir, password_name, keyring_backend):
+        delete_password(cluster_dir, password_name, keyring_backend)
+        sys.exit(0)
+    else:
+        sys.exit(2)
+
+
+if __name__ == "__main__":
+    from sys import argv
+
+    main(*argv[1:])

architectures/lib/generate-vault

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# © Copyright EnterpriseDB UK Limited 2015-2023 - All rights reserved.
+
+import sys
+from os import path
+from tpaexec.password import store_password, exists, generate_password
+
+
+def main(
+    cluster_dir, keyring_backend=None, password_name="vault_pass",
+):
+    """
+    generate and store vault password in chosen keyring backend
+    exit code 0 if new password was generated else exit code 2 if password already existed.
+    """
+    if not exists(path.basename(path.abspath(cluster_dir)), password_name, keyring_backend):
+        store_password(
+            cluster_dir,
+            password_name,
+            generate_password(),
+            keyring_backend,
+        )
+        sys.exit(0)
+    else:
+        sys.exit(2)
+
+
+if __name__ == "__main__":
+    from sys import argv
+
+    main(*argv[1:])

architectures/lib/use-vault

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# © Copyright EnterpriseDB UK Limited 2015-2023 - All rights reserved.
+from os import getenv, getcwd
+from os import path
+
+import yaml
+from tpaexec.password import show_password
+
+
+def main():
+    """
+    retrieve vault password from chosen keyring backend
+    """
+    cluster_dir = getcwd()
+    try:
+        with open(path.join(cluster_dir, "config.yml")) as config_file:
+            config = yaml.safe_load(config_file)
+            keyring_backend = config.get(
+                "keyring_backend", getenv("TPA_KEYRING_BACKEND", None)
+            )
+            password_name = config.get(
+                "vault_name", "vault_pass")
+    except IOError:
+        print(f"could not load {path.join(cluster_dir, 'config.yml')}")
+        raise
+
+    show_password(cluster_dir, password_name, keyring_backend)
+
+
+if __name__ == "__main__":
+    main()

bin/tpaexec

@@ -352,8 +352,13 @@ provision() {
 }
 
 cmd() {
+    # first ensure that vault_pass is defined for the cluster
+    # otherwise don't add it to the args
+    if "${TPA_DIR}/architectures/lib/use-vault" >> /dev/null 2>&1; then
+        args=--vault-password-file="${TPA_DIR}/architectures/lib/use-vault"
+    fi
     $ansible \
-        -i inventory --vault-password-file vault/vault_pass.txt \
+        -i inventory "$args"\
         -e cluster_dir="$(pwd)" "$@"
 }
 
@@ -369,8 +374,8 @@ playbook() {
         args+=(-i inventory)
     fi
 
-    if [ -f vault/vault_pass.txt ]; then
-        args+=(--vault-password-file vault/vault_pass.txt)
+    if "${TPA_DIR}/architectures/lib/use-vault" >> /dev/null 2>&1; then
+        args+=(--vault-password-file "${TPA_DIR}/architectures/lib/use-vault")
     fi
 
     "$ansible"-playbook "${args[@]}" "$@"
@@ -422,7 +427,7 @@ try_as_playbook_or_script() {
             if [[ -x $exec ]]; then
                 case $command in
                     store-password|show-password)
-                        _with_ansible_env "$exec" "$@" "--vault_password_file=vault/vault_pass.txt"
+                        _with_ansible_env "$exec" "$@" "--vault_password_file=${TPA_DIR}/architectures/lib/use-vault"
                         return $?
                     ;;
                     *)
@@ -611,6 +616,9 @@ Available help topics:
     store-password
                 Commands to manage passwords for cluster users
                 (e.g., postgres_password, pgbouncer_password)
+    show-vault
+                Command to show the vault password of the cluster
+                This is used to encrypt other passwords.
 
   Miscellaneous
 
@@ -1156,14 +1164,14 @@ case "$command" in
     cmd|ping|provision|deploy|upgrade|playbook|deprovision)
         time $command "$@"
         real_exit_status=$?
-        playbook ${TPA_DIR}/architectures/lib/commands/check-repositories.yml
+        playbook "${TPA_DIR}/architectures/lib/commands/check-repositories.yml"
         exit $real_exit_status
         ;;
 
     *)
         try_as_playbook_or_script "$@"
         real_exit_status=$?
-        playbook ${TPA_DIR}/architectures/lib/commands/check-repositories.yml
+        playbook "${TPA_DIR}/architectures/lib/commands/check-repositories.yml"
         exit $real_exit_status
         ;;
 esac

docs/src/tpaexec-configure.md

@@ -395,7 +395,7 @@ Use the `--use-ansible-tower` and `--tower-git-repository` options to
 create a cluster adapted for deployment with Ansible Tower. See [Ansible
 Tower](tower.md) for details.
 
-## git repository
+## Git repository
 
 By default, a git repository is created with an initial branch named
 after the cluster, and a single commit is made, with the configure
@@ -406,6 +406,34 @@ option. (Note that in an Ansible Tower cluster, a git repository is
 required and will be created later by `tpaexec provision` if it does not
 already exist.)
 
+## Keyring backend for vault password
+
+TPA generates a cluster specific ansible vault password.
+This password is used to encrypt other sensitive variables generated
+for the cluster, postgres user password, barman user password and so on.
+
+Keyring backend `system` will leverage the best keyring backend on your system
+from the list of supported backend by python keyring module including
+gnome-keyring and secret-tool.
+
+Default is to store the vault password using `system` keyring for new cluster.
+removing `keyring_backend: system` in config.yml file **before** any `provision`
+will revert previous default to store vault password in plaintext file.
+
+Using `keyring_backend: system` also generates a `vault_name` entry in config.yml
+used to store the vault password unique storage name. TPA generate an UUID by
+default but there is no naming scheme requirements.
+
+Note: When using `keyring_backend: system` and the same base config.yml file
+for multiple clusters with same `cluster_name`, by copying the config file to
+a different location, ensure the value pair (`vault_name`, `cluster_name`)
+is unique for each cluster copy.
+
+Note: When using `keyring_backend: system` and moving an already provisioned
+cluster folder to a different tpa host, ensure that you export the associated
+vault password on the new machine's system keyring. vault password can be
+displayed via `tpaexec show-vault <cluster_dir>`.
+
 ## Examples
 
 Let's see what happens when we run the following command:

lib/filter_plugins/filters.py

@@ -265,7 +265,7 @@ def cmdline(playbook_dir):
         and args[5] == "-i"
         and args[6] == "inventory"
         and args[7] == "--vault-password-file"
-        and args[8] == "vault/vault_pass.txt"
+        and args[8].endswith("use-vault")
     ):
         tpaexec = "tpaexec"