diff --git a/.github/workflows/dsf_poc_cli.yml b/.github/workflows/dsf_poc_cli.yml index c66ea49df..eea682dd1 100644 --- a/.github/workflows/dsf_poc_cli.yml +++ b/.github/workflows/dsf_poc_cli.yml @@ -132,11 +132,6 @@ jobs: run: | find ./examples/ -type f -exec sed -i -f sed.expr {} \; - - name: Sets env vars for environment - run: | - echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV - if: github.ref != 'refs/heads/"master"' - - name: Create terraform backend file run: | cat << EOF > $EXAMPLE_DIR/backend.tf diff --git a/.github/workflows/dsf_poc_cli_azure.yml b/.github/workflows/dsf_poc_cli_azure.yml index 4b1e5847b..8dc920e80 100644 --- a/.github/workflows/dsf_poc_cli_azure.yml +++ b/.github/workflows/dsf_poc_cli_azure.yml @@ -129,11 +129,6 @@ jobs: run: | find $EXAMPLE_DIR -type f -exec sed -i -f sed.expr {} \; - - name: Sets env vars for environment - run: | - echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV - if: github.ref != 'refs/heads/"master"' - - name: Create terraform backend file run: | cat << EOF > $EXAMPLE_DIR/backend.tf diff --git a/.github/workflows/dsf_single_account_cli.yml b/.github/workflows/dsf_single_account_cli.yml index a705588c6..70895b7dd 100644 --- a/.github/workflows/dsf_single_account_cli.yml +++ b/.github/workflows/dsf_single_account_cli.yml @@ -98,11 +98,6 @@ jobs: run: | find ./examples/ -type f -exec sed -i -f sed.expr {} \; - - name: Sets env vars for environment - run: | - echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV - if: inputs.branch != 'master' - - name: Set Workspace run: | if [ ${{ inputs.branch }} == 'master' ] || [ ${{ inputs.branch }} == 'dev' ]; then diff --git a/.github/workflows/nightly_sonar_poc_basic_cli.yml b/.github/workflows/nightly_sonar_poc_basic_cli.yml index 3b38b238a..3894f0a68 100644 --- a/.github/workflows/nightly_sonar_poc_basic_cli.yml +++ b/.github/workflows/nightly_sonar_poc_basic_cli.yml @@ -66,11 +66,6 @@ jobs: echo '${{ toJSON(github) }}' find ./examples/ -type f -exec sed -i -f sed.expr {} \; - - name: Sets env vars for environment - run: | - echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV - if: github.ref != 'refs/heads/"master"' - - name: Create terraform backend file run: | cat << EOF > $EXAMPLE_DIR/backend.tf diff --git a/.github/workflows/sonar_multi_account_cli.yml b/.github/workflows/sonar_multi_account_cli.yml index 5eef224d6..b0526b641 100644 --- a/.github/workflows/sonar_multi_account_cli.yml +++ b/.github/workflows/sonar_multi_account_cli.yml @@ -65,11 +65,6 @@ jobs: run: | find ./examples/ -type f -exec sed -i -f sed.expr {} \; - - name: Sets env vars for environment - run: | - echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV - if: inputs.branch != 'master' - - name: Create terraform backend file run: | cat << EOF > $EXAMPLE_DIR/backend.tf diff --git a/.github/workflows/sonar_poc_cli.yml b/.github/workflows/sonar_poc_cli.yml index 47d013f15..b88c8a729 100644 --- a/.github/workflows/sonar_poc_cli.yml +++ b/.github/workflows/sonar_poc_cli.yml @@ -104,11 +104,6 @@ jobs: run: | find $EXAMPLE_DIR -type f -exec sed -i -f sed.expr {} \; - - name: Sets env vars for environment - run: | - echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV - if: github.ref != 'refs/heads/"master"' - - name: Create terraform backend file run: | cat << EOF > $EXAMPLE_DIR/backend.tf diff --git a/.github/workflows/sonar_poc_cloud.yml b/.github/workflows/sonar_poc_cloud.yml index f5cb6e2f4..18339938e 100644 --- a/.github/workflows/sonar_poc_cloud.yml +++ b/.github/workflows/sonar_poc_cloud.yml @@ -47,11 +47,6 @@ jobs: run: | find ./examples/ -type f -exec sed -i -f sed.expr {} \; - - name: Sets env vars for environment - run: | - echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV - if: github.ref != 'refs/heads/"master"' - # Install the latest version of Terraform CLI and configure the Terraform CLI configuration file with a Terraform Cloud user API token - name: Setup Terraform uses: hashicorp/setup-terraform@v2 diff --git a/.github/workflows/sonar_upgrade.yml b/.github/workflows/sonar_upgrade.yml index 3b2b875f5..01e5cf1c9 100644 --- a/.github/workflows/sonar_upgrade.yml +++ b/.github/workflows/sonar_upgrade.yml @@ -101,11 +101,6 @@ jobs: run: | find ${{ env.DEPLOYMENT_EXAMPLE_DIR }} -type f -exec sed -i -f sed.expr {} \; - - name: Sets env vars for environment - run: | - echo "TF_VAR_tarball_s3_bucket=0ed58e18-0c0c-11ed-861d-0242ac120003" >> $GITHUB_ENV - if: github.ref != 'refs/heads/"master"' - - name: Create terraform backend file for the deployment run: | cat << EOF > $DEPLOYMENT_EXAMPLE_DIR/backend.tf diff --git a/modules/aws/agentless-gw/variables.tf b/modules/aws/agentless-gw/variables.tf index 6c64ac78e..6e062a6a5 100644 --- a/modules/aws/agentless-gw/variables.tf +++ b/modules/aws/agentless-gw/variables.tf @@ -126,7 +126,7 @@ variable "binaries_location" { s3_region = string s3_key = string }) - description = "S3 DSF installation location. If tarball_url not set, binaries_location is used" + description = "S3 DSF installation location. If tarball_url not set, binaries_location is used. For example, { s3_bucket = 'my-bucket', s3_region = 'us-west-2', s3_key = 'installation-files/my-file'}. This means that the path to the installation file is s3://my-bucket/installation-files/my-file" default = { s3_bucket = "" s3_region = "" diff --git a/modules/aws/core/globals/main.tf b/modules/aws/core/globals/main.tf index 948a81be6..5d906336a 100644 --- a/modules/aws/core/globals/main.tf +++ b/modules/aws/core/globals/main.tf @@ -25,8 +25,8 @@ locals { } sonar_supported_versions = keys(local.sonar_tarball_s3_key_map) sonar_fully_supported_versions = setsubtract(local.sonar_supported_versions, ["4.9", "4.10.0.0", "4.10.0.1", "4.10"]) - s3_object = var.tarball_s3_key != null ? var.tarball_s3_key : local.sonar_tarball_s3_key_map[var.sonar_version] - s3_object_version = regex("\\d\\.\\d*", local.s3_object) + sonar_installation_s3_key = var.installation_s3_key != null ? var.installation_s3_key : join("/", [local.sonar_installation_s3_prefix, local.sonar_tarball_s3_key_map[var.sonar_version]]) + sonar_s3_object_version = regex("\\d\\.\\d*", local.sonar_installation_s3_key) } locals { @@ -50,6 +50,11 @@ locals { dra_version = lookup(local.dra_version_map, var.dra_version, var.dra_version) } +locals { + sonar_installation_s3_prefix = "sonar" + dam_agent_installation_s3_prefix = "dam-agent" +} + resource "random_id" "salt" { byte_length = 2 } diff --git a/modules/aws/core/globals/outputs.tf b/modules/aws/core/globals/outputs.tf index a1b78fb0f..600572ef0 100644 --- a/modules/aws/core/globals/outputs.tf +++ b/modules/aws/core/globals/outputs.tf @@ -42,18 +42,19 @@ output "tags" { output "tarball_location" { value = { - s3_bucket = var.tarball_s3_bucket.bucket - s3_region = var.tarball_s3_bucket.region - s3_key = local.s3_object - version = local.s3_object_version + s3_bucket = var.installation_s3_bucket.bucket + s3_region = var.installation_s3_bucket.region + s3_key = local.sonar_installation_s3_key + version = local.sonar_s3_object_version } } output "dam_agent_installation_location" { value = { - s3_bucket = var.tarball_s3_bucket.bucket - s3_region = var.tarball_s3_bucket.region - s3_key = null + s3_bucket = var.installation_s3_bucket.bucket + s3_region = var.installation_s3_bucket.region + s3_prefix = local.dam_agent_installation_s3_prefix + s3_object = null } } diff --git a/modules/aws/core/globals/variables.tf b/modules/aws/core/globals/variables.tf index 80a9eead0..7ac10552c 100644 --- a/modules/aws/core/globals/variables.tf +++ b/modules/aws/core/globals/variables.tf @@ -4,7 +4,7 @@ variable "sonar_version" { description = "The Sonar version to install. Supported versions are: 4.9 and up. Both long and short version formats are supported, for example, 4.12.0.10 or 4.12. The short format maps to the latest patch." } -variable "tarball_s3_bucket" { +variable "installation_s3_bucket" { type = object({ bucket = string region = string @@ -13,13 +13,13 @@ variable "tarball_s3_bucket" { bucket = "1ef8de27-ed95-40ff-8c08-7969fc1b7901" region = "us-east-1" } - description = "S3 bucket containing the installation tarballs. Use default to get Imperva's bucket" + description = "S3 bucket containing the installation files for the EDF Hub, Agentless Gateway or POC DAM Agent. Use default to get Imperva's bucket." } -variable "tarball_s3_key" { +variable "installation_s3_key" { type = string + description = "Name and prefix of the installation file of the EDF Hub, Agentless Gateway or POC DAM Agent in the S3 bucket. Keep empty if you prefer to use the sonar_version variable." default = null - description = "Name of the installation file in s3 bucket. Keep empty if you prefer to use the sonar_version variable" } variable "dra_version" { diff --git a/modules/aws/db-with-agent/iam_role.tf b/modules/aws/db-with-agent/iam_role.tf index f77b97a57..be89ee016 100644 --- a/modules/aws/db-with-agent/iam_role.tf +++ b/modules/aws/db-with-agent/iam_role.tf @@ -22,15 +22,13 @@ locals { "Version" : "2012-10-17", "Statement" : [ { - "Sid" : "VisualEditor0", + "Sid" : "getFileFromS3BucketPrefix", "Effect" : "Allow", "Action" : [ - "s3:GetObject", - "s3:ListBucket" + "s3:GetObject" ] "Resource" : [ - "arn:aws:s3:::${local.binaries_location.s3_bucket}", - "arn:aws:s3:::${local.binaries_location.s3_bucket}/*", + "arn:aws:s3:::${local.installation_s3_bucket_and_prefix}/*", ] } ] diff --git a/modules/aws/db-with-agent/main.tf b/modules/aws/db-with-agent/main.tf index 498c15915..31b3be71d 100644 --- a/modules/aws/db-with-agent/main.tf +++ b/modules/aws/db-with-agent/main.tf @@ -4,11 +4,10 @@ locals { db_type = var.db_type != null ? var.db_type : random_shuffle.db.result[0] os_type = var.os_type != null ? var.os_type : random_shuffle.os.result[0] - binaries_location = { - s3_bucket = var.binaries_location.s3_bucket - s3_region = var.binaries_location.s3_region - s3_key = var.binaries_location.s3_key != null ? var.binaries_location.s3_key : local.os_params[local.os_type].image_name - } + + installation_s3_object = var.binaries_location.s3_object != null ? var.binaries_location.s3_object : local.os_params[local.os_type].installation_filename + installation_s3_key = var.binaries_location.s3_prefix != null ? join("/", [var.binaries_location.s3_prefix, local.installation_s3_object]) : local.installation_s3_object + installation_s3_bucket_and_prefix = var.binaries_location.s3_prefix != null ? join("/", [var.binaries_location.s3_bucket, var.binaries_location.s3_prefix]) : var.binaries_location.s3_bucket } resource "random_shuffle" "db" { diff --git a/modules/aws/db-with-agent/os_params.tf b/modules/aws/db-with-agent/os_params.tf index 8272c74c0..d53f24f25 100644 --- a/modules/aws/db-with-agent/os_params.tf +++ b/modules/aws/db-with-agent/os_params.tf @@ -5,7 +5,7 @@ locals { ami_name = "RHEL-8.6.0_HVM-2022*-x86_64-2-Hourly2-GP2" ami_ssh_user = "ec2-user" agent_installation_dir = "/opt/imperva", - image_name = "Imperva-ragent-RHEL-v8-kSMP-px86_64-b14.6.0.60.0.637577.bsx" + installation_filename = "Imperva-ragent-RHEL-v8-kSMP-px86_64-b14.6.0.60.0.637577.bsx" package_install = <<-EOF yum update -y yum install unzip -y @@ -37,7 +37,7 @@ locals { ami_name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*" ami_ssh_user = "ubuntu" agent_installation_dir = "/usr/imperva", - image_name = "Imperva-ragent-UBN-px86_64-b14.6.0.60.0.636085.bsx" + installation_filename = "Imperva-ragent-UBN-px86_64-b14.6.0.60.0.636085.bsx" package_install = <<-EOF apt update -y apt install unzip diff --git a/modules/aws/db-with-agent/userdata.tf b/modules/aws/db-with-agent/userdata.tf index 0d6be39b5..d84c03463 100644 --- a/modules/aws/db-with-agent/userdata.tf +++ b/modules/aws/db-with-agent/userdata.tf @@ -16,11 +16,12 @@ locals { sudo ./aws/install export PATH=$PATH:/usr/local/bin:/usr/local/bin echo "Downloading agent:" - aws s3 cp s3://${local.binaries_location.s3_bucket}/${local.binaries_location.s3_key} . - chmod +x ./${local.binaries_location.s3_key} + INSTALLATION_FILE=${local.installation_s3_object} + aws s3 cp s3://${var.binaries_location.s3_bucket}/${local.installation_s3_key} . --region ${var.binaries_location.s3_region} + chmod +x ./"$INSTALLATION_FILE" echo "Installing agent:" - ./${local.binaries_location.s3_key} -n -d ${local.agent_installation_dir} - rm ${local.binaries_location.s3_key} + ./"$INSTALLATION_FILE" -n -d ${local.agent_installation_dir} + rm "$INSTALLATION_FILE" echo "Registering agent:" ${local.agent_installation_dir}/ragent/bin/cli --dcfg ${local.agent_installation_dir}/ragent/etc --dtarget ${local.agent_installation_dir}/ragent/etc --dlog ${local.agent_installation_dir}/ragent/etc/logs/cli registration advanced-register registration-type=Primary is-db-agent=true tunnel-protocol=TCP gw-ip=${var.registration_params.agent_gateway_host} gw-port=443 manual-settings-activation=Automatic monitor-network-channels=Both password="${var.registration_params.secure_password}" ragent-name="${join("-", [var.friendly_name, random_id.salt.hex])}" site='${var.registration_params.site}' server-group="${var.registration_params.server_group}"; echo "Starting agent:" diff --git a/modules/aws/db-with-agent/variables.tf b/modules/aws/db-with-agent/variables.tf index 165a386ad..de900ad26 100644 --- a/modules/aws/db-with-agent/variables.tf +++ b/modules/aws/db-with-agent/variables.tf @@ -79,9 +79,10 @@ variable "binaries_location" { type = object({ s3_bucket = string s3_region = string - s3_key = string + s3_prefix = string + s3_object = string }) - description = "S3 DSF DAM agent installation location" + description = "S3 DSF DAM agent installation location. For example, { s3_bucket = 'my-bucket', s3_region = 'us-west-2', s3_prefix = 'installation-files', s3_object = 'my-file' }. This means that the path to the installation file is s3://my-bucket/installation-files/my-file. If you don't have a prefix or an object, keep them null." nullable = false } diff --git a/modules/aws/hub/variables.tf b/modules/aws/hub/variables.tf index 8eee16e14..80e81ef07 100644 --- a/modules/aws/hub/variables.tf +++ b/modules/aws/hub/variables.tf @@ -146,7 +146,7 @@ variable "binaries_location" { s3_region = string s3_key = string }) - description = "S3 DSF installation location. If tarball_url not set, binaries_location is used" + description = "S3 DSF installation location. If tarball_url not set, binaries_location is used. For example, { s3_bucket = 'my-bucket', s3_region = 'us-west-2', s3_key = 'installation-files/my-file'}. This means that the path to the installation file is s3://my-bucket/installation-files/my-file" default = { s3_bucket = "" s3_region = "" diff --git a/modules/aws/sonar-base-instance/iam_role.tf b/modules/aws/sonar-base-instance/iam_role.tf index 08679e465..bd44df1ee 100644 --- a/modules/aws/sonar-base-instance/iam_role.tf +++ b/modules/aws/sonar-base-instance/iam_role.tf @@ -43,15 +43,13 @@ locals { "Version" : "2012-10-17", "Statement" : [ { - "Sid" : "VisualEditor0", + "Sid" : "getFileFromS3BucketPrefix", "Effect" : "Allow", "Action" : [ - "s3:GetObject", - "s3:ListBucket" + "s3:GetObject" ] "Resource" : [ - "arn:aws:s3:::${var.binaries_location.s3_bucket}", - "arn:aws:s3:::${var.binaries_location.s3_bucket}/*", + "arn:aws:s3:::${local.installation_s3_bucket_and_prefix}/*", ] } ] diff --git a/modules/aws/sonar-base-instance/main.tf b/modules/aws/sonar-base-instance/main.tf index 684e2cb31..cebec04a6 100644 --- a/modules/aws/sonar-base-instance/main.tf +++ b/modules/aws/sonar-base-instance/main.tf @@ -17,6 +17,11 @@ locals { security_group_ids = concat( [for sg in aws_security_group.dsf_base_sg : sg.id], var.security_group_ids) + + # For example, if the binaries_location.s3_key is "path/to/file.zip", then the installation_s3_prefix will be "path/to" + # If the binaries_location.s3_key is "file.zip", then the installation_s3_prefix will be null + installation_s3_prefix = try(regex("^(.*)/[^/]+", var.binaries_location.s3_key)[0], null) + installation_s3_bucket_and_prefix = local.installation_s3_prefix != null ? join("/", [var.binaries_location.s3_bucket, local.installation_s3_prefix]) : var.binaries_location.s3_bucket } resource "aws_eip" "dsf_instance_eip" { diff --git a/modules/aws/sonar-base-instance/variables.tf b/modules/aws/sonar-base-instance/variables.tf index 8992aec6c..b96da1ce2 100644 --- a/modules/aws/sonar-base-instance/variables.tf +++ b/modules/aws/sonar-base-instance/variables.tf @@ -139,7 +139,7 @@ variable "binaries_location" { s3_region = string s3_key = string }) - description = "S3 DSF installation location. If tarball_url not set, binaries_location is used." + description = "S3 DSF installation location. If tarball_url not set, binaries_location is used. For example, { s3_bucket = 'my-bucket', s3_region = 'us-west-2', s3_key = 'installation-files/my-file'}. This means that the path to the installation file is s3://my-bucket/installation-files/my-file" default = { s3_bucket = "" s3_region = "" diff --git a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py index 899a6675f..8e2d8ee8f 100644 --- a/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py +++ b/modules/aws/sonar-upgrader/python_upgrader/upgrade/main.py @@ -18,6 +18,8 @@ POSTFLIGHT_VALIDATIONS_SCRIPT_NAME = "run_postflight_validations.py" CLEAN_OLD_DEPLOYMENTS_SCRIPT_NAME = "clean_old_deployments.sh" +SONAR_INSTALLATION_S3_PREFIX = "sonar" + UNDEFINED_PYTHON_LOCATION = "UNDEFINED_PYTHON_LOCATION" # Globals @@ -732,7 +734,7 @@ def get_upgrade_script_args(target_version, tarball_location): def get_tarball_s3_key(target_version): - return f"jsonar-{target_version}.tar.gz" + return f"{SONAR_INSTALLATION_S3_PREFIX}/jsonar-{target_version}.tar.gz" def maybe_run_postflight_validations(extended_node, target_version, script_file_name, stop_on_failure,