feat: Delete data dump from S3 after data load has been successful

buildspec.database-copy.yml

@@ -0,0 +1,24 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - echo "3.9" > .python-version
+      - cd images/tools/database-copy/
+      - echo Login to Amazon ECR
+      - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/uktrade
+      - SHORT_HASH=$(git rev-parse --short HEAD)
+
+  build:
+    commands:
+      - echo "Build database-copy ($SHORT_HASH) started on $(date)"
+      - docker build --tag public.ecr.aws/uktrade/database-copy:tag-latest .
+      - docker tag public.ecr.aws/uktrade/database-copy:tag-latest public.ecr.aws/uktrade/database-copy:$SHORT_HASH
+      - echo "Build database-copy ($SHORT_HASH) completed on $(date)"
+
+  post_build:
+    commands:
+      - echo "Push database-copy ($SHORT_HASH) started on $(date)"
+      - docker push public.ecr.aws/uktrade/database-copy:tag-latest
+      - docker push public.ecr.aws/uktrade/database-copy:$SHORT_HASH
+      - echo "Push database-copy ($SHORT_HASH) completed on $(date)"

dbt_platform_helper/COMMANDS.md

@@ -42,6 +42,7 @@
 - [platform-helper database](#platform-helper-database)
 - [platform-helper database dump](#platform-helper-database-dump)
 - [platform-helper database load](#platform-helper-database-load)
+- [platform-helper database copy](#platform-helper-database-copy)
 - [platform-helper version](#platform-helper-version)
 - [platform-helper version get-platform-helper-for-project](#platform-helper-version-get-platform-helper-for-project)
 
@@ -962,10 +963,12 @@ platform-helper notify add-comment <slack_channel_id> <slack_token>
 
 [↩ Parent](#platform-helper)
 
+    Commands to copy data between databases.
+
 ## Usage
 
 ```
-platform-helper database (dump|load) 
+platform-helper database (dump|load|copy) 
 ```
 
 ## Options
@@ -975,6 +978,7 @@ platform-helper database (dump|load)
 
 ## Commands
 
+- [`copy` ↪](#platform-helper-database-copy)
 - [`dump` ↪](#platform-helper-database-dump)
 - [`load` ↪](#platform-helper-database-load)
 
@@ -987,23 +991,20 @@ platform-helper database (dump|load)
 ## Usage
 
 ```
-platform-helper database dump --account-id <account_id> --app <application> 
-                              --env <environment> --database <database> 
-                              --vpc-name <vpc_name> 
+platform-helper database dump --from <from_env> --database <database> 
+                              [--app <application>] [--from-vpc <from_vpc>] 
 ```
 
 ## Options
 
-- `--account-id <text>`
-
 - `--app <text>`
-
-- `--env <text>`
-
+  - The application name. Required unless you are running the command from your deploy repo
+- `--from <text>`
+  - This is required unless you are running the command from your deploy repo
 - `--database <text>`
-
-- `--vpc-name <text>`
-
+  - The name of the database you are dumping data from
+- `--from-vpc <text>`
+  - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
@@ -1016,22 +1017,62 @@ platform-helper database dump --account-id <account_id> --app <application>
 ## Usage
 
 ```
-platform-helper database load --account-id <account_id> --app <application> 
-                              --env <environment> --database <database> 
-                              --vpc-name <vpc_name> 
+platform-helper database load --to <to_env> --database <database> 
+                              [--app <application>] [--to-vpc <to_vpc>] 
+                              [--auto-approve] 
 ```
 
 ## Options
 
-- `--account-id <text>`
-
 - `--app <text>`
+  - The application name. Required unless you are running the command from your deploy repo
+- `--to <text>`
+  - The environment you are loading data into
+- `--database <text>`
+  - The name of the database you are loading data into
+- `--to-vpc <text>`
+  - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
+- `--auto-approve <boolean>` _Defaults to False._
 
-- `--env <text>`
+- `--help <boolean>` _Defaults to False._
+  - Show this message and exit.
+
+# platform-helper database copy
+
+[↩ Parent](#platform-helper-database)
+
+    Copy a database between environments.
+
+## Usage
 
+```
+platform-helper database copy --from <from_env> --to <to_env> --database <database> 
+                              --svc <service> [--app <application>] [--from-vpc <from_vpc>] 
+                              [--to-vpc <to_vpc>] [--template (default|migration|dmas-migration)] 
+                              [--auto-approve] [--no-maintenance-page] 
+```
+
+## Options
+
+- `--app <text>`
+  - The application name. Required unless you are running the command from your deploy repo
+- `--from <text>`
+  - The environment you are copying data from
+- `--to <text>`
+  - The environment you are copying data into
 - `--database <text>`
+  - The name of the database you are copying
+- `--from-vpc <text>`
+  - The vpc the environment you are copying from is running in. Required unless you are running the command from your deploy repo
+- `--to-vpc <text>`
+  - The vpc the environment you are copying into is running in. Required unless you are running the command from your deploy repo
+- `--auto-approve <boolean>` _Defaults to False._
 
-- `--vpc-name <text>`
+- `--svc <text>` _Defaults to ['web']._
+
+- `--template <choice>` _Defaults to default._
+  - The maintenance page you wish to put up.
+- `--no-maintenance-page <boolean>` _Defaults to False._
 
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.

dbt_platform_helper/commands/database.py

@@ -1,33 +1,112 @@
 import click
 
-from dbt_platform_helper.commands.database_helpers import DatabaseCopy
+from dbt_platform_helper.commands.environment import AVAILABLE_TEMPLATES
+from dbt_platform_helper.domain.database_copy import DatabaseCopy
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 
 
 @click.group(chain=True, cls=ClickDocOptGroup)
 def database():
-    pass
+    """Commands to copy data between databases."""
 
 
 @database.command(name="dump")
-@click.option("--account-id", type=str, required=True)
-@click.option("--app", type=str, required=True)
-@click.option("--env", type=str, required=True)
-@click.option("--database", type=str, required=True)
-@click.option("--vpc-name", type=str, required=True)
-def dump(account_id, app, env, database, vpc_name):
+@click.option(
+    "--app",
+    type=str,
+    help="The application name. Required unless you are running the command from your deploy repo",
+)
+@click.option(
+    "--from",
+    "from_env",
+    type=str,
+    required=True,
+    help="This is required unless you are running the command from your deploy repo",
+)
+@click.option(
+    "--database", type=str, required=True, help="The name of the database you are dumping data from"
+)
+@click.option(
+    "--from-vpc",
+    type=str,
+    help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
+)
+def dump(app, from_env, database, from_vpc):
     """Dump a database into an S3 bucket."""
-    data_copy = DatabaseCopy(account_id, app, env, database, vpc_name)
-    data_copy.dump()
+    data_copy = DatabaseCopy(app, database)
+    data_copy.dump(from_env, from_vpc)
 
 
 @database.command(name="load")
-@click.option("--account-id", type=str, required=True)
-@click.option("--app", type=str, required=True)
-@click.option("--env", type=str, required=True)
-@click.option("--database", type=str, required=True)
-@click.option("--vpc-name", type=str, required=True)
-def load(account_id, app, env, database, vpc_name):
+@click.option(
+    "--app",
+    type=str,
+    help="The application name. Required unless you are running the command from your deploy repo",
+)
+@click.option(
+    "--to", "to_env", type=str, required=True, help="The environment you are loading data into"
+)
+@click.option(
+    "--database", type=str, required=True, help="The name of the database you are loading data into"
+)
+@click.option(
+    "--to-vpc",
+    type=str,
+    help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
+)
+@click.option("--auto-approve/--no-auto-approve", default=False)
+def load(app, to_env, database, to_vpc, auto_approve):
     """Load a database from an S3 bucket."""
-    data_copy = DatabaseCopy(account_id, app, env, database, vpc_name)
-    data_copy.load()
+    data_copy = DatabaseCopy(app, database, auto_approve)
+    data_copy.load(to_env, to_vpc)
+
+
+@database.command(name="copy")
+@click.option(
+    "--app",
+    type=str,
+    help="The application name. Required unless you are running the command from your deploy repo",
+)
+@click.option(
+    "--from", "from_env", type=str, required=True, help="The environment you are copying data from"
+)
+@click.option(
+    "--to", "to_env", type=str, required=True, help="The environment you are copying data into"
+)
+@click.option(
+    "--database", type=str, required=True, help="The name of the database you are copying"
+)
+@click.option(
+    "--from-vpc",
+    type=str,
+    help="The vpc the environment you are copying from is running in. Required unless you are running the command from your deploy repo",
+)
+@click.option(
+    "--to-vpc",
+    type=str,
+    help="The vpc the environment you are copying into is running in. Required unless you are running the command from your deploy repo",
+)
+@click.option("--auto-approve/--no-auto-approve", default=False)
+@click.option("--svc", type=str, required=True, multiple=True, default=["web"])
+@click.option(
+    "--template",
+    type=click.Choice(AVAILABLE_TEMPLATES),
+    default="default",
+    help="The maintenance page you wish to put up.",
+)
+@click.option("--no-maintenance-page", flag_value=True)
+def copy(
+    app,
+    from_env,
+    to_env,
+    database,
+    from_vpc,
+    to_vpc,
+    auto_approve,
+    svc,
+    template,
+    no_maintenance_page,
+):
+    """Copy a database between environments."""
+    data_copy = DatabaseCopy(app, database, auto_approve)
+    data_copy.copy(from_env, to_env, from_vpc, to_vpc, svc, template, no_maintenance_page)