From ef05fbc515b70e32d828b5c12a2eecff31dd826f Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Fri, 26 Feb 2021 17:22:50 +0530 Subject: [PATCH] added tests for: 1. rule filtering 2. resource skipping 3. remote types --- test/e2e/help/help_test.go | 20 +- test/e2e/init/init_test.go | 20 +- test/e2e/scan/config/invalid_severity.toml | 2 + test/e2e/scan/config/scan_and_skip_rules.toml | 9 + .../kubernetes_file_resource_skipping.txt | 29 +++ .../terraform_file_resource_skipping.txt | 142 ++++++++++++ .../rules_filtering/scan_and_skip_rules.txt | 39 ++++ .../rules_filtering/scan_multiple_rules.txt | 105 +++++++++ .../rules_filtering/scan_single_rule.txt | 28 +++ .../rules_filtering/skip_multiple_rules.txt | 28 +++ .../rules_filtering/skip_single_rule.txt | 105 +++++++++ test/e2e/scan/scan_config_only_test.go | 29 +-- test/e2e/scan/scan_k8s_files_test.go | 16 +- test/e2e/scan/scan_remote_test.go | 187 ++++++++++++++++ test/e2e/scan/scan_rules_filtering_test.go | 204 ++++++++++++++++++ test/e2e/scan/scan_test.go | 170 ++++++++++++++- test/e2e/scan/scan_tf_files_test.go | 34 +-- test/e2e/scan/scan_utils.go | 10 +- .../resource_skipping/kubernetes/config.yaml | 10 + .../iac/resource_skipping/terraform/main.tf | 105 +++++++++ test/helper/helper.go | 17 ++ 21 files changed, 1234 insertions(+), 75 deletions(-) create mode 100644 test/e2e/scan/config/invalid_severity.toml create mode 100644 test/e2e/scan/config/scan_and_skip_rules.toml create mode 100644 test/e2e/scan/golden/resource_skipping/kubernetes_file_resource_skipping.txt create mode 100644 test/e2e/scan/golden/resource_skipping/terraform_file_resource_skipping.txt create mode 100644 test/e2e/scan/golden/rules_filtering/scan_and_skip_rules.txt create mode 100644 test/e2e/scan/golden/rules_filtering/scan_multiple_rules.txt create mode 100644 test/e2e/scan/golden/rules_filtering/scan_single_rule.txt create mode 100644 test/e2e/scan/golden/rules_filtering/skip_multiple_rules.txt create mode 100644 test/e2e/scan/golden/rules_filtering/skip_single_rule.txt create mode 100644 test/e2e/scan/scan_remote_test.go create mode 100644 test/e2e/scan/scan_rules_filtering_test.go create mode 100644 test/e2e/test_data/iac/resource_skipping/kubernetes/config.yaml create mode 100644 test/e2e/test_data/iac/resource_skipping/terraform/main.tf diff --git a/test/e2e/help/help_test.go b/test/e2e/help/help_test.go index cad5cc263..0798009f0 100644 --- a/test/e2e/help/help_test.go +++ b/test/e2e/help/help_test.go @@ -6,7 +6,6 @@ import ( helpUtils "github.com/accurics/terrascan/test/e2e/help" "github.com/accurics/terrascan/test/helper" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" ) @@ -39,21 +38,21 @@ var _ = Describe("Help", func() { Describe("terrascan is run without any command", func() { It("should print all supported commands and exit with status code 0", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter) - helpUtils.ValidateExitCodeAndOutput(session, 0, "golden/help_command.txt", true) + helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_command.txt", true) }) }) Describe("terrascan is run -h flag", func() { It("should print all supported commands and exit with status code 0", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "-h") - helpUtils.ValidateExitCodeAndOutput(session, 0, "golden/help_flag.txt", true) + helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_flag.txt", true) }) }) Describe("terrascan is run with an unkonwn command", func() { It("should exit with status code 1 and display a error message", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "test") - helpUtils.ValidateExitCodeAndOutput(session, 1, "golden/incorrect_command.txt", false) + helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeOne, "golden/incorrect_command.txt", false) }) }) @@ -61,43 +60,42 @@ var _ = Describe("Help", func() { Context("with no arguments", func() { It("should print the terrascan help and exit with status code 0", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand) - helpUtils.ValidateExitCodeAndOutput(session, 0, "golden/help_command.txt", true) - Eventually(session).Should(gexec.Exit(0)) + helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_command.txt", true) }) }) Context("for init command", func() { It("should print help for init and exit with status code 0", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "init") - helpUtils.ValidateExitCodeAndOutput(session, 0, "golden/help_init.txt", true) + helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_init.txt", true) }) }) Context("for scan command", func() { It("should print help for init and exit with status code 0", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "scan") - helpUtils.ValidateExitCodeAndOutput(session, 0, "golden/help_scan.txt", true) + helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_scan.txt", true) }) }) Context("for server command", func() { It("should print help for init and exit with status code 0", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "server") - helpUtils.ValidateExitCodeAndOutput(session, 0, "golden/help_server.txt", true) + helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_server.txt", true) }) }) Context("for version command", func() { It("should print help for init and exit with status code 0", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "version") - helpUtils.ValidateExitCodeAndOutput(session, 0, "golden/help_version.txt", true) + helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_version.txt", true) }) }) Context("for an unkonwn command", func() { It("should display that help topic is not available for entered command and exit with status code 0", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, helpCommand, "test") - helpUtils.ValidateExitCodeAndOutput(session, 0, "golden/help_unsupported_command.txt", false) + helpUtils.ValidateExitCodeAndOutput(session, helper.ExitCodeZero, "golden/help_unsupported_command.txt", false) }) }) }) diff --git a/test/e2e/init/init_test.go b/test/e2e/init/init_test.go index 5297333ca..efcd9a368 100644 --- a/test/e2e/init/init_test.go +++ b/test/e2e/init/init_test.go @@ -48,7 +48,7 @@ var _ = Describe("Init", func() { Describe("terrascan init is run", func() { When("without any flags", func() { It("should download policies and exit with status code 0", func() { - session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, helper.ExitCodeZero) Expect(outWriter).Should(gbytes.Say("")) }) @@ -82,7 +82,7 @@ var _ = Describe("Init", func() { It("should exit with status code 0", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, initCommand, "-h") - Eventually(session).Should(gexec.Exit(0)) + Eventually(session).Should(gexec.Exit(helper.ExitCodeZero)) }) }) @@ -96,7 +96,7 @@ var _ = Describe("Init", func() { It("should exit with status code 1", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "inti") - Eventually(session, 5).Should(gexec.Exit(1)) + Eventually(session, 5).Should(gexec.Exit(helper.ExitCodeOne)) }) }) @@ -118,7 +118,7 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should error out and exit with status code 1", func() { - session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 1) + session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, helper.ExitCodeOne) helper.ContainsErrorSubString(session, `failed to download policies. error: 'Get "https://repository/url/info/refs?service=git-upload-pack": dial tcp:`) }) }) @@ -130,7 +130,7 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should error out and exit with status code 1", func() { - session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 1) + session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, helper.ExitCodeOne) helper.ContainsErrorSubString(session, `failed to initialize terrascan. error : failed to checkout git branch 'invalid-branch'. error: 'reference not found'`) }) }) @@ -142,7 +142,7 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should error out and exit with status code 1", func() { - session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 1) + session = initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, helper.ExitCodeOne) helper.ContainsErrorSubString(session, "invalid/path: no such file or directory") }) }) @@ -154,7 +154,7 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("should download policies and exit with status code 0", func() { - initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, helper.ExitCodeZero) }) }) Context("the config file has valid data", func() { @@ -166,7 +166,7 @@ var _ = Describe("Init", func() { os.Setenv(terrascanConfigEnvName, "") }) It("init should download the repo provided in the config file", func() { - initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, helper.ExitCodeZero) }) Context("Kai Monkey git repo is downloaded", func() { It("should validate Kai Monkey repo in the policy path", func() { @@ -183,14 +183,14 @@ var _ = Describe("Init", func() { Context("running init the first time", func() { var modifiedTime time.Time It("should download policies at the default policy path", func() { - initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, helper.ExitCodeZero) fi, err := os.Stat(defaultPolicyRepoPath) Expect(err).ToNot(HaveOccurred()) modifiedTime = fi.ModTime() }) Context("running init the second time", func() { It("should download policies again at the default policy path", func() { - initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, 0) + initUtil.RunInitCommand(terrascanBinaryPath, outWriter, errWriter, helper.ExitCodeZero) fi, err := os.Stat(defaultPolicyRepoPath) Expect(err).ToNot(HaveOccurred()) Expect(fi.ModTime()).To(BeTemporally(">", modifiedTime)) diff --git a/test/e2e/scan/config/invalid_severity.toml b/test/e2e/scan/config/invalid_severity.toml new file mode 100644 index 000000000..372869b23 --- /dev/null +++ b/test/e2e/scan/config/invalid_severity.toml @@ -0,0 +1,2 @@ +[severity] +level = "test" \ No newline at end of file diff --git a/test/e2e/scan/config/scan_and_skip_rules.toml b/test/e2e/scan/config/scan_and_skip_rules.toml new file mode 100644 index 000000000..9a88d5e4f --- /dev/null +++ b/test/e2e/scan/config/scan_and_skip_rules.toml @@ -0,0 +1,9 @@ +[rules] + scan-rules = [ + "AWS.RDS.DS.High.1041", + "AWS.AWS RDS.NS.High.0101", + "AWS.RDS.DataSecurity.High.0577" + ] + skip-rules = [ + "AWS.RDS.DataSecurity.High.0577" + ] \ No newline at end of file diff --git a/test/e2e/scan/golden/resource_skipping/kubernetes_file_resource_skipping.txt b/test/e2e/scan/golden/resource_skipping/kubernetes_file_resource_skipping.txt new file mode 100644 index 000000000..dcb436250 --- /dev/null +++ b/test/e2e/scan/golden/resource_skipping/kubernetes_file_resource_skipping.txt @@ -0,0 +1,29 @@ +{ + "results": { + "violations": null, + "skipped_violations": [ + { + "rule_name": "noOwnerLabel", + "description": "No owner for namespace affects the operations", + "rule_id": "AC-K8-OE-NS-L-0128", + "severity": "LOW", + "category": "Operational Efficiency", + "skip_comment": "reason to skip the rule", + "resource_name": "production", + "resource_type": "kubernetes_namespace", + "file": "/Users/apple/go/src/github.com/patilpankaj212/terrascan/test/e2e/test_data/iac/resource_skipping/kubernetes/config.yaml", + "line": 1 + } + ], + "scan_summary": { + "file/folder": "/Users/apple/go/src/github.com/patilpankaj212/terrascan/test/e2e/test_data/iac/resource_skipping/kubernetes", + "iac_type": "k8s", + "scanned_at": "2021-02-26 17:22:16.050445 +0000 UTC", + "policies_validated": 53, + "violated_policies": 0, + "low": 0, + "medium": 0, + "high": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/scan/golden/resource_skipping/terraform_file_resource_skipping.txt b/test/e2e/scan/golden/resource_skipping/terraform_file_resource_skipping.txt new file mode 100644 index 000000000..e2f46c6e9 --- /dev/null +++ b/test/e2e/scan/golden/resource_skipping/terraform_file_resource_skipping.txt @@ -0,0 +1,142 @@ +{ + "results": { + "violations": [ + { + "rule_name": "ec2UsingIMDSv1", + "description": "EC2 instances should disable IMDS or require IMDSv2 as this can be related to the weaponization phase of kill chain", + "rule_id": "AC-AWS-NS-IN-M-1172", + "severity": "MEDIUM", + "category": "Network Security", + "resource_name": "instanceWithIMDv1_emptyblock", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 70 + }, + { + "rule_name": "ec2UsingIMDSv1", + "description": "EC2 instances should disable IMDS or require IMDSv2 as this can be related to the weaponization phase of kill chain", + "rule_id": "AC-AWS-NS-IN-M-1172", + "severity": "MEDIUM", + "category": "Network Security", + "resource_name": "instanceWithIMDv1_fullblock", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 77 + }, + { + "rule_name": "ec2UsingIMDSv1", + "description": "EC2 instances should disable IMDS or require IMDSv2 as this can be related to the weaponization phase of kill chain", + "rule_id": "AC-AWS-NS-IN-M-1172", + "severity": "MEDIUM", + "category": "Network Security", + "resource_name": "instanceWithIMDv1_endpoint_not_present", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 98 + }, + { + "rule_name": "instanceWithNoVpc", + "description": "Ensure that your AWS application is not deployed within the default Virtual Private Cloud in order to follow security best practices", + "rule_id": "AC-AW-IS-IN-M-0144", + "severity": "MEDIUM", + "category": "Infrastructure Security", + "resource_name": "instanceWithNoVpc", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 51 + }, + { + "rule_name": "instanceWithNoVpc", + "description": "Ensure that your AWS application is not deployed within the default Virtual Private Cloud in order to follow security best practices", + "rule_id": "AC-AW-IS-IN-M-0144", + "severity": "MEDIUM", + "category": "Infrastructure Security", + "resource_name": "instanceWithIMDv1_emptyblock", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 70 + }, + { + "rule_name": "instanceWithNoVpc", + "description": "Ensure that your AWS application is not deployed within the default Virtual Private Cloud in order to follow security best practices", + "rule_id": "AC-AW-IS-IN-M-0144", + "severity": "MEDIUM", + "category": "Infrastructure Security", + "resource_name": "instanceWithIMDv1_fullblock", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 77 + }, + { + "rule_name": "instanceWithNoVpc", + "description": "Ensure that your AWS application is not deployed within the default Virtual Private Cloud in order to follow security best practices", + "rule_id": "AC-AW-IS-IN-M-0144", + "severity": "MEDIUM", + "category": "Infrastructure Security", + "resource_name": "instanceWithIMDv1_endpoint_not_present", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 98 + }, + { + "rule_name": "instanceWithNoVpc", + "description": "Ensure that your AWS application is not deployed within the default Virtual Private Cloud in order to follow security best practices", + "rule_id": "AC-AW-IS-IN-M-0144", + "severity": "MEDIUM", + "category": "Infrastructure Security", + "resource_name": "instanceWithPublicIp", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 61 + } + ], + "skipped_violations": [ + { + "rule_name": "ec2UsingIMDSv1", + "description": "EC2 instances should disable IMDS or require IMDSv2 as this can be related to the weaponization phase of kill chain", + "rule_id": "AC-AWS-NS-IN-M-1172", + "severity": "MEDIUM", + "category": "Network Security", + "skip_comment": "should skip this rule", + "resource_name": "instanceWithPublicIp", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 61 + }, + { + "rule_name": "ec2UsingIMDSv1", + "description": "EC2 instances should disable IMDS or require IMDSv2 as this can be related to the weaponization phase of kill chain", + "rule_id": "AC-AWS-NS-IN-M-1172", + "severity": "MEDIUM", + "category": "Network Security", + "skip_comment": "need to skip this rule", + "resource_name": "instanceWithIMDv1_token_not_present", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 87 + }, + { + "rule_name": "instanceWithNoVpc", + "description": "Ensure that your AWS application is not deployed within the default Virtual Private Cloud in order to follow security best practices", + "rule_id": "AC-AW-IS-IN-M-0144", + "severity": "MEDIUM", + "category": "Infrastructure Security", + "skip_comment": "can skip this rule", + "resource_name": "instanceWithIMDv1_token_not_present", + "resource_type": "aws_instance", + "file": "main.tf", + "line": 87 + } + ], + "scan_summary": { + "file/folder": "/Users/apple/go/src/github.com/patilpankaj212/terrascan/test/e2e/test_data/iac/resource_skipping/terraform", + "iac_type": "terraform", + "scanned_at": "2021-02-26 17:05:56.671863 +0000 UTC", + "policies_validated": 53, + "violated_policies": 8, + "low": 0, + "medium": 8, + "high": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/scan/golden/rules_filtering/scan_and_skip_rules.txt b/test/e2e/scan/golden/rules_filtering/scan_and_skip_rules.txt new file mode 100644 index 000000000..588740b9b --- /dev/null +++ b/test/e2e/scan/golden/rules_filtering/scan_and_skip_rules.txt @@ -0,0 +1,39 @@ +{ + "results": { + "violations": [ + { + "rule_name": "rdsAutoMinorVersionUpgradeEnabled", + "description": "RDS Instance Auto Minor Version Upgrade flag disabled", + "rule_id": "AWS.RDS.DS.High.1041", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + }, + { + "rule_name": "rdsPubliclyAccessible", + "description": "RDS Instance publicly_accessible flag is true", + "rule_id": "AWS.AWS RDS.NS.High.0101", + "severity": "HIGH", + "category": "Network Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + } + ], + "skipped_violations": null, + "scan_summary": { + "file/folder": "/Users/apple/go/src/github.com/patilpankaj212/terrascan/test/e2e/test_data/iac/aws/aws_db_instance_violation", + "iac_type": "terraform", + "scanned_at": "2021-02-26 15:13:04.249604 +0000 UTC", + "policies_validated": 2, + "violated_policies": 2, + "low": 0, + "medium": 0, + "high": 2 + } + } +} \ No newline at end of file diff --git a/test/e2e/scan/golden/rules_filtering/scan_multiple_rules.txt b/test/e2e/scan/golden/rules_filtering/scan_multiple_rules.txt new file mode 100644 index 000000000..29ed5be33 --- /dev/null +++ b/test/e2e/scan/golden/rules_filtering/scan_multiple_rules.txt @@ -0,0 +1,105 @@ +{ + "results": { + "violations": [ + { + "rule_name": "rdsIamAuthEnabled", + "description": "Ensure that your RDS database has IAM Authentication enabled.", + "rule_id": "AWS.RDS.DataSecurity.High.0577", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi4", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 55 + }, + { + "rule_name": "rdsIamAuthEnabled", + "description": "Ensure that your RDS database has IAM Authentication enabled.", + "rule_id": "AWS.RDS.DataSecurity.High.0577", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi6", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 82 + }, + { + "rule_name": "rdsIamAuthEnabled", + "description": "Ensure that your RDS database has IAM Authentication enabled.", + "rule_id": "AWS.RDS.DataSecurity.High.0577", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi2", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 25 + }, + { + "rule_name": "rdsIamAuthEnabled", + "description": "Ensure that your RDS database has IAM Authentication enabled.", + "rule_id": "AWS.RDS.DataSecurity.High.0577", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + }, + { + "rule_name": "rdsIamAuthEnabled", + "description": "Ensure that your RDS database has IAM Authentication enabled.", + "rule_id": "AWS.RDS.DataSecurity.High.0577", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi5", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 69 + }, + { + "rule_name": "rdsIamAuthEnabled", + "description": "Ensure that your RDS database has IAM Authentication enabled.", + "rule_id": "AWS.RDS.DataSecurity.High.0577", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + }, + { + "rule_name": "rdsAutoMinorVersionUpgradeEnabled", + "description": "RDS Instance Auto Minor Version Upgrade flag disabled", + "rule_id": "AWS.RDS.DS.High.1041", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + }, + { + "rule_name": "rdsPubliclyAccessible", + "description": "RDS Instance publicly_accessible flag is true", + "rule_id": "AWS.AWS RDS.NS.High.0101", + "severity": "HIGH", + "category": "Network Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + } + ], + "skipped_violations": null, + "scan_summary": { + "file/folder": "/Users/apple/go/src/github.com/patilpankaj212/terrascan/test/e2e/test_data/iac/aws/aws_db_instance_violation", + "iac_type": "terraform", + "scanned_at": "2021-02-26 14:48:58.697828 +0000 UTC", + "policies_validated": 3, + "violated_policies": 8, + "low": 0, + "medium": 0, + "high": 8 + } + } +} \ No newline at end of file diff --git a/test/e2e/scan/golden/rules_filtering/scan_single_rule.txt b/test/e2e/scan/golden/rules_filtering/scan_single_rule.txt new file mode 100644 index 000000000..bf0e2365e --- /dev/null +++ b/test/e2e/scan/golden/rules_filtering/scan_single_rule.txt @@ -0,0 +1,28 @@ +{ + "results": { + "violations": [ + { + "rule_name": "rdsAutoMinorVersionUpgradeEnabled", + "description": "RDS Instance Auto Minor Version Upgrade flag disabled", + "rule_id": "AWS.RDS.DS.High.1041", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + } + ], + "skipped_violations": null, + "scan_summary": { + "file/folder": "/Users/apple/go/src/github.com/patilpankaj212/terrascan/test/e2e/test_data/iac/aws/aws_db_instance_violation", + "iac_type": "terraform", + "scanned_at": "2021-02-26 14:39:52.574497 +0000 UTC", + "policies_validated": 1, + "violated_policies": 1, + "low": 0, + "medium": 0, + "high": 1 + } + } +} \ No newline at end of file diff --git a/test/e2e/scan/golden/rules_filtering/skip_multiple_rules.txt b/test/e2e/scan/golden/rules_filtering/skip_multiple_rules.txt new file mode 100644 index 000000000..806dda5ba --- /dev/null +++ b/test/e2e/scan/golden/rules_filtering/skip_multiple_rules.txt @@ -0,0 +1,28 @@ +{ + "results": { + "violations": [ + { + "rule_name": "rdsPubliclyAccessible", + "description": "RDS Instance publicly_accessible flag is true", + "rule_id": "AWS.AWS RDS.NS.High.0101", + "severity": "HIGH", + "category": "Network Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + } + ], + "skipped_violations": null, + "scan_summary": { + "file/folder": "/Users/apple/go/src/github.com/patilpankaj212/terrascan/test/e2e/test_data/iac/aws/aws_db_instance_violation", + "iac_type": "terraform", + "scanned_at": "2021-02-26 15:08:44.344294 +0000 UTC", + "policies_validated": 50, + "violated_policies": 1, + "low": 0, + "medium": 0, + "high": 1 + } + } +} \ No newline at end of file diff --git a/test/e2e/scan/golden/rules_filtering/skip_single_rule.txt b/test/e2e/scan/golden/rules_filtering/skip_single_rule.txt new file mode 100644 index 000000000..902c712ad --- /dev/null +++ b/test/e2e/scan/golden/rules_filtering/skip_single_rule.txt @@ -0,0 +1,105 @@ +{ + "results": { + "violations": [ + { + "rule_name": "rdsAutoMinorVersionUpgradeEnabled", + "description": "RDS Instance Auto Minor Version Upgrade flag disabled", + "rule_id": "AWS.RDS.DS.High.1041", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + }, + { + "rule_name": "rdsHasStorageEncrypted", + "description": "Ensure that your RDS database instances encrypt the underlying storage. Encrypted RDS instances use the industry standard AES-256 encryption algorithm to encrypt data on the server that hosts RDS DB instances. After data is encrypted, RDS handles authentication of access and descryption of data transparently with minimal impact on performance.", + "rule_id": "AWS.RDS.DataSecurity.High.0414", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi6", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 82 + }, + { + "rule_name": "rdsHasStorageEncrypted", + "description": "Ensure that your RDS database instances encrypt the underlying storage. Encrypted RDS instances use the industry standard AES-256 encryption algorithm to encrypt data on the server that hosts RDS DB instances. After data is encrypted, RDS handles authentication of access and descryption of data transparently with minimal impact on performance.", + "rule_id": "AWS.RDS.DataSecurity.High.0414", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi2", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 25 + }, + { + "rule_name": "rdsHasStorageEncrypted", + "description": "Ensure that your RDS database instances encrypt the underlying storage. Encrypted RDS instances use the industry standard AES-256 encryption algorithm to encrypt data on the server that hosts RDS DB instances. After data is encrypted, RDS handles authentication of access and descryption of data transparently with minimal impact on performance.", + "rule_id": "AWS.RDS.DataSecurity.High.0414", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi4", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 55 + }, + { + "rule_name": "rdsHasStorageEncrypted", + "description": "Ensure that your RDS database instances encrypt the underlying storage. Encrypted RDS instances use the industry standard AES-256 encryption algorithm to encrypt data on the server that hosts RDS DB instances. After data is encrypted, RDS handles authentication of access and descryption of data transparently with minimal impact on performance.", + "rule_id": "AWS.RDS.DataSecurity.High.0414", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi1", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 10 + }, + { + "rule_name": "rdsHasStorageEncrypted", + "description": "Ensure that your RDS database instances encrypt the underlying storage. Encrypted RDS instances use the industry standard AES-256 encryption algorithm to encrypt data on the server that hosts RDS DB instances. After data is encrypted, RDS handles authentication of access and descryption of data transparently with minimal impact on performance.", + "rule_id": "AWS.RDS.DataSecurity.High.0414", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + }, + { + "rule_name": "rdsHasStorageEncrypted", + "description": "Ensure that your RDS database instances encrypt the underlying storage. Encrypted RDS instances use the industry standard AES-256 encryption algorithm to encrypt data on the server that hosts RDS DB instances. After data is encrypted, RDS handles authentication of access and descryption of data transparently with minimal impact on performance.", + "rule_id": "AWS.RDS.DataSecurity.High.0414", + "severity": "HIGH", + "category": "Data Security", + "resource_name": "PtShGgAdi5", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 69 + }, + { + "rule_name": "rdsPubliclyAccessible", + "description": "RDS Instance publicly_accessible flag is true", + "rule_id": "AWS.AWS RDS.NS.High.0101", + "severity": "HIGH", + "category": "Network Security", + "resource_name": "PtShGgAdi3", + "resource_type": "aws_db_instance", + "file": "main.tf", + "line": 39 + } + ], + "skipped_violations": null, + "scan_summary": { + "file/folder": "/Users/apple/go/src/github.com/patilpankaj212/terrascan/test/e2e/test_data/iac/aws/aws_db_instance_violation", + "iac_type": "terraform", + "scanned_at": "2021-02-26 15:05:50.968554 +0000 UTC", + "policies_validated": 52, + "violated_policies": 8, + "low": 0, + "medium": 0, + "high": 8 + } + } +} \ No newline at end of file diff --git a/test/e2e/scan/scan_config_only_test.go b/test/e2e/scan/scan_config_only_test.go index 09949e445..31a11e095 100644 --- a/test/e2e/scan/scan_config_only_test.go +++ b/test/e2e/scan/scan_config_only_test.go @@ -3,6 +3,7 @@ package scan_test import ( "path/filepath" + scanUtils "github.com/accurics/terrascan/test/e2e/scan" "github.com/accurics/terrascan/test/helper" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -35,8 +36,8 @@ var _ = Describe("Scan With Config Only Flag", func() { Context("it doesn't support --config-only flag", func() { Context("human readable output format is the default output format", func() { It("should result in an error and exit with status code 1", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir, "--config-only") - Eventually(session).Should(gexec.Exit(1)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir, "--config-only") + Eventually(session).Should(gexec.Exit(helper.ExitCodeOne)) helper.ContainsErrorSubString(session, "please use yaml or json output format when using --config-only flag") }) }) @@ -46,8 +47,8 @@ var _ = Describe("Scan With Config Only Flag", func() { When("output type is xml", func() { Context("it doesn't support --config-only flag", func() { It("should result in an error and exit with status code 1", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir, "--config-only", "-o", "xml") - Eventually(session, 5).Should(gexec.Exit(1)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir, "--config-only", "-o", "xml") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) helper.ContainsErrorSubString(session, "failed to write XML output. error: 'xml: unsupported type: output.AllResourceConfigs'") }) }) @@ -56,8 +57,8 @@ var _ = Describe("Scan With Config Only Flag", func() { When("output type is junit-xml", func() { Context("it doesn't support --config-only flag", func() { It("should result in an error and exit with status code 1", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir, "--config-only", "-o", "junit-xml") - Eventually(session, 5).Should(gexec.Exit(1)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir, "--config-only", "-o", "junit-xml") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) helper.ContainsErrorSubString(session, "incorrect input for JunitXML writer, supported type is policy.EngineOutput") }) }) @@ -69,8 +70,8 @@ var _ = Describe("Scan With Config Only Flag", func() { When("output type is json", func() { Context("it supports --config-only flag", func() { It("should display config json and exit with status code 3", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir, "--config-only", "-o", "json") - Eventually(session, 5).Should(gexec.Exit(3)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir, "--config-only", "-o", "json") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) }) }) }) @@ -78,8 +79,8 @@ var _ = Describe("Scan With Config Only Flag", func() { When("output type is yaml", func() { Context("it supports --config-only flag", func() { It("should display config json and exit with status code 3", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir, "--config-only", "-o", "yaml") - Eventually(session, 5).Should(gexec.Exit(3)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir, "--config-only", "-o", "yaml") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) }) }) }) @@ -92,8 +93,8 @@ var _ = Describe("Scan With Config Only Flag", func() { When("output type is json", func() { Context("it supports --config-only flag", func() { It("should display config json and exit with status code 3", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir, "--config-only", "-o", "json", "-i", "k8s") - Eventually(session, 5).Should(gexec.Exit(3)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir, "--config-only", "-o", "json", "-i", "k8s") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) }) }) }) @@ -101,8 +102,8 @@ var _ = Describe("Scan With Config Only Flag", func() { When("output type is yaml", func() { Context("it supports --config-only flag", func() { It("should display config json and exit with status code 3", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir, "--config-only", "-o", "yaml", "-i", "k8s") - Eventually(session, 5).Should(gexec.Exit(3)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir, "--config-only", "-o", "yaml", "-i", "k8s") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) }) }) }) diff --git a/test/e2e/scan/scan_k8s_files_test.go b/test/e2e/scan/scan_k8s_files_test.go index 4dab24d3c..37ccf54fd 100644 --- a/test/e2e/scan/scan_k8s_files_test.go +++ b/test/e2e/scan/scan_k8s_files_test.go @@ -40,8 +40,8 @@ var _ = Describe("Scan is run for k8s files", func() { Context("iac type k8s is not default", func() { When("k8s files are scanned but iac type is not specified", func() { It("should print error related to terraform files not being present", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir) - Eventually(session, 2).Should(gexec.Exit(1)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) helper.ContainsErrorSubString(session, "has no terraform config files") }) }) @@ -50,41 +50,41 @@ var _ = Describe("Scan is run for k8s files", func() { Context("iac type is specified as k8s", func() { It("should scan and display violations in human output format", func() { scanArgs := []string{"-i", "k8s", "-p", policyDir, "-d", iacDir} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_human.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_human.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) When("-v flag is used for verbose output", func() { It("should display verbose output for human output format", func() { scanArgs := []string{"-i", "k8s", "-p", policyDir, "-d", iacDir, "-v"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_human_verbose.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_human_verbose.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is json", func() { It("should display violations in json format", func() { scanArgs := []string{"-i", "k8s", "-p", policyDir, "-d", iacDir, "-o", "json"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_json.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_json.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is yaml", func() { It("should display violations in yaml format", func() { scanArgs := []string{"-i", "k8s", "-p", policyDir, "-d", iacDir, "-o", "yaml"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_yaml.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_yaml.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is xml", func() { It("should display violations in xml format", func() { scanArgs := []string{"-i", "k8s", "-p", policyDir, "-d", iacDir, "-o", "xml"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_xml.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_xml.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is junit-xml", func() { It("should display violations in junit-xml format", func() { scanArgs := []string{"-i", "k8s", "-p", policyDir, "-d", iacDir, "-o", "junit-xml"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_junit_xml.txt", 3, true, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/k8s_scans/k8s/kubernetes_ingress_violations/kubernetes_ingress_junit_xml.txt", helper.ExitCodeThree, true, true, outWriter, errWriter, scanArgs...) }) }) }) diff --git a/test/e2e/scan/scan_remote_test.go b/test/e2e/scan/scan_remote_test.go new file mode 100644 index 000000000..5a13519b1 --- /dev/null +++ b/test/e2e/scan/scan_remote_test.go @@ -0,0 +1,187 @@ +package scan_test + +import ( + scanUtils "github.com/accurics/terrascan/test/e2e/scan" + "github.com/accurics/terrascan/test/helper" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gexec" +) + +var _ = Describe("Scan Command using remote types", func() { + BeforeEach(func() { + outWriter = gbytes.NewBuffer() + errWriter = gbytes.NewBuffer() + }) + + AfterEach(func() { + outWriter = nil + errWriter = nil + }) + + Context("remote type is supplied, but remote URL is not", func() { + errString := "empty remote url or type or desitnation dir path" + + When("remote type is git", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "git") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + }) + }) + + When("remote type is s3", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "s3") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + }) + }) + + When("remote type is gcs", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "gcs") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + }) + }) + + When("remote type is http", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "http") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + }) + }) + + When("remote type is terraform-registry", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "terraform-registry") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + }) + }) + }) + + Context("valid remote type is supplied with invalid remote URL ", func() { + invalidRemoteURL := "test" + When("remote type is git", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "git", "--remote-url", invalidRemoteURL) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + }) + }) + + When("remote type is s3", func() { + remoteURL := invalidRemoteURL + JustBeforeEach(func() { + invalidRemoteURL = "s3://" + }) + JustAfterEach(func() { + invalidRemoteURL = remoteURL + }) + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "s3", "--remote-url", invalidRemoteURL) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + }) + }) + + When("remote type is gcs", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "gcs", "--remote-url", invalidRemoteURL) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + }) + }) + + When("remote type is http", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "http", "--remote-url", invalidRemoteURL) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + }) + }) + + When("remote type is terraform-registry", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", invalidRemoteURL) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + }) + }) + + Context("when remote type is unsupported", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "unsupportedType", "--remote-url", invalidRemoteURL) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, "supplied remote type is not supported") + }) + }) + }) + + Context("valid remote type is supplied with valid remote URL", func() { + When("remote type is git", func() { + remoteURL := "github.com/accurics/KaiMonkey/terraform/aws" + It("should download the resource and generate scan results", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "git", "--remote-url", remoteURL) + Eventually(session, 10).Should(gexec.Exit(helper.ExitCodeThree)) + }) + }) + + When("remote type is s3", func() { + It("should download the resource and generate scan results", func() { + Skip("Skipping this test until we have a s3 url") + }) + }) + + When("remote type is gcs", func() { + It("should download the resource and generate scan results", func() { + Skip("Skipping this test until we have a gcs url") + }) + }) + + When("remote type is http", func() { + It("should download the resource and generate scan results", func() { + Skip("Skipping this test") + }) + }) + + When("remote type is terraform-registry", func() { + remoteURL := "terraform-aws-modules/vpc/aws" + When("terraform registry URL doesn't have version specified, it downloads the latest available version", func() { + It("should download the resource and generate scan results", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL) + // has a OR condition because we don't know if there would be violations or not + Eventually(session, 10).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero))) + }) + }) + + When("terraform registry remote url has a version", func() { + oldRemoteURL := remoteURL + JustBeforeEach(func() { + remoteURL = "terraform-aws-modules/vpc/aws:2.22.0" + }) + JustAfterEach(func() { + remoteURL = oldRemoteURL + }) + It("should download the remote registry and generate scan results", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL) + // has a OR condition because we don't know if there would be violations or not + Eventually(session, 10).Should(Or(gexec.Exit(helper.ExitCodeThree), gexec.Exit(helper.ExitCodeZero))) + }) + }) + + When("terraform registry remote url has a invalid version", func() { + oldRemoteURL := remoteURL + JustBeforeEach(func() { + remoteURL = "terraform-aws-modules/vpc/aws:blah" + }) + JustAfterEach(func() { + remoteURL = oldRemoteURL + }) + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-r", "terraform-registry", "--remote-url", remoteURL) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + }) + }) + }) + }) +}) diff --git a/test/e2e/scan/scan_rules_filtering_test.go b/test/e2e/scan/scan_rules_filtering_test.go new file mode 100644 index 000000000..9ef2e97a8 --- /dev/null +++ b/test/e2e/scan/scan_rules_filtering_test.go @@ -0,0 +1,204 @@ +package scan_test + +import ( + "os" + "path/filepath" + + scanUtils "github.com/accurics/terrascan/test/e2e/scan" + "github.com/accurics/terrascan/test/helper" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gexec" +) + +var ( + terrascanConfigEnvName string = "TERRASCAN_CONFIG" + severityLevelIncorrectError string = "severity level not supported" +) + +var _ = Describe("Scan command with rule filtering options", func() { + + BeforeEach(func() { + outWriter = gbytes.NewBuffer() + errWriter = gbytes.NewBuffer() + }) + + AfterEach(func() { + outWriter = nil + errWriter = nil + }) + + var policyDir, iacDir string + var err error + + iacDir, err = filepath.Abs("../test_data/iac/aws/aws_db_instance_violation") + It("should not error out while getting absolute path", func() { + Expect(err).NotTo(HaveOccurred()) + }) + + policyDir, err = filepath.Abs("../test_data/policies/") + It("should not error out while getting absolute path", func() { + Expect(err).NotTo(HaveOccurred()) + }) + + Describe("rule filtering via command line options", func() { + + Context("--scan-rules options is used", func() { + Context("single rule is specified via --scan-rules option", func() { + It("should scan only the rules specified in --scan-rules option", func() { + scanRules := "AWS.RDS.DS.High.1041" + scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json", "--scan-rules", scanRules} + scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/rules_filtering/scan_single_rule.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) + }) + }) + Context("multiple rules are specified via --scan-rules option", func() { + It("should scan only the rules specified in --scan-rules option", func() { + scanRules := "AWS.RDS.DS.High.1041,AWS.AWS RDS.NS.High.0101,AWS.RDS.DataSecurity.High.0577" + scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json", "--scan-rules", scanRules} + scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/rules_filtering/scan_multiple_rules.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) + }) + }) + }) + + Context("--skip-rules options is used", func() { + Context("single rule is specified via --skip-rules option", func() { + It("should skip the rule specified in --skip-rules option", func() { + skipRules := "AWS.RDS.DataSecurity.High.0577" + scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json", "--skip-rules", skipRules} + scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/rules_filtering/skip_single_rule.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) + }) + }) + Context("multiple rules are specified via --skip-rules option", func() { + It("should skip the rules specified in --skip-rules option", func() { + skipRules := "AWS.RDS.DS.High.1041,AWS.RDS.DataSecurity.High.0414,AWS.RDS.DataSecurity.High.0577" + scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json", "--skip-rules", skipRules} + scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/rules_filtering/skip_multiple_rules.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) + }) + }) + }) + + Context("both --scan-rules and --skip-rules are specified", func() { + Context("single rule is specified via --skip-rules option", func() { + It("should scan and skip the rules as specified with --scan-rules and --skip-rules option", func() { + scanRules := "AWS.RDS.DS.High.1041,AWS.AWS RDS.NS.High.0101,AWS.RDS.DataSecurity.High.0577" + skipRules := "AWS.RDS.DataSecurity.High.0577" + scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json", "--skip-rules", skipRules, "--scan-rules", scanRules} + scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/rules_filtering/scan_and_skip_rules.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) + }) + }) + }) + + Context("severity level is specified", func() { + When("severity specified is invalid", func() { + It("should error out and exit with status code 1", func() { + args := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--severity", "test"} + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, args...) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, severityLevelIncorrectError) + }) + }) + + When("valid severity level is specified", func() { + oldIacDir := iacDir + JustBeforeEach(func() { + iacDir, err = filepath.Abs("../test_data/iac/aws/aws_ami_violation") + }) + + JustAfterEach(func() { + iacDir = oldIacDir + }) + Context("severity leve specified is 'low'", func() { + Context("iac file has only medium severity violations", func() { + It("should report the violations and exit with status code 3", func() { + args := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--severity", "low"} + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, args...) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) + }) + }) + }) + Context("severity leve specified is 'high'", func() { + Context("iac files has only medium severity violations", func() { + It("should not report any violation and exit with status code 0", func() { + args := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-o", "json", "--severity", "high"} + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, args...) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero)) + }) + }) + }) + }) + }) + }) + + Describe("rule filtering via config file", func() { + Context("config file is specified using -c flag", func() { + Context("both scan and skip rules are specified", func() { + It("should scan and skip the rules as specified with --scan-rules and --skip-rules option", func() { + configFileAbsPath, err := filepath.Abs("config/scan_and_skip_rules.toml") + Expect(err).NotTo(HaveOccurred()) + scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json", "-c", configFileAbsPath} + scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/rules_filtering/scan_and_skip_rules.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) + }) + }) + }) + + Context("config file is specified using TERRASCAN_CONFIG env variable", func() { + Context("both scan and skip rules are specified", func() { + JustBeforeEach(func() { + os.Setenv(terrascanConfigEnvName, "config/scan_and_skip_rules.toml") + }) + JustAfterEach(func() { + os.Setenv(terrascanConfigEnvName, "") + }) + It("should scan and skip the rules as specified with --scan-rules and --skip-rules option", func() { + Skip("skipping this test due to https://github.com/accurics/terrascan/issues/570, should be implemented when fixed") + }) + }) + + Context("invalid severity is specified in config file", func() { + It("should error out and exit with status code 1", func() { + configFileAbsPath, err := filepath.Abs("config/invalid_severity.toml") + Expect(err).NotTo(HaveOccurred()) + + args := []string{scanUtils.ScanCommand, "-p", policyDir, "-d", iacDir, "-c", configFileAbsPath} + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, args...) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, severityLevelIncorrectError) + }) + }) + }) + }) + + Describe("resource specific rule skipping", func() { + Context("resource skipping in tf files", func() { + oldIacDir := iacDir + JustBeforeEach(func() { + iacDir, err = filepath.Abs("../test_data/iac/resource_skipping/terraform") + }) + JustAfterEach(func() { + iacDir = oldIacDir + }) + It("should display violations, skipped violations and exit with status code 3", func() { + scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json"} + scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/resource_skipping/terraform_file_resource_skipping.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) + }) + }) + + Context("resource skipping in k8s files", func() { + oldIacDir := iacDir + JustBeforeEach(func() { + iacDir, err = filepath.Abs("../test_data/iac/resource_skipping/kubernetes") + }) + JustAfterEach(func() { + iacDir = oldIacDir + }) + + // the iac file has only one resource with one violation, which is skipped. + // hence, the exit code is 0 + It("should display skipped violations and exit with status code 0", func() { + scanArgs := []string{"-p", policyDir, "-d", iacDir, "-i", "k8s", "-o", "json"} + scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/resource_skipping/kubernetes_file_resource_skipping.txt", helper.ExitCodeZero, false, true, outWriter, errWriter, scanArgs...) + }) + }) + }) +}) diff --git a/test/e2e/scan/scan_test.go b/test/e2e/scan/scan_test.go index 372939dd8..46bde0502 100644 --- a/test/e2e/scan/scan_test.go +++ b/test/e2e/scan/scan_test.go @@ -1,9 +1,12 @@ package scan_test import ( + "fmt" "io" + "os" "path/filepath" + scanUtils "github.com/accurics/terrascan/test/e2e/scan" "github.com/accurics/terrascan/test/helper" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -12,7 +15,6 @@ import ( ) var ( - scanComand string = "scan" session *gexec.Session terrascanBinaryPath string outWriter, errWriter io.Writer @@ -36,8 +38,8 @@ var _ = Describe("Scan", func() { Describe("scan command is run with -h flag", func() { It("should print help for scan and exit with status code 0", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-h") - Eventually(session).Should(gexec.Exit(0)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-h") + Eventually(session).Should(gexec.Exit(helper.ExitCodeZero)) goldenFileAbsPath, err := filepath.Abs("golden/scan_help.txt") Expect(err).NotTo(HaveOccurred()) helper.CompareActualWithGolden(session, goldenFileAbsPath, true) @@ -47,7 +49,7 @@ var _ = Describe("Scan", func() { Describe("typo in the scan command, eg: scna", func() { It("should exit with status code 1", func() { session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, "scna") - Eventually(session).Should(gexec.Exit(1)) + Eventually(session).Should(gexec.Exit(helper.ExitCodeOne)) }) It("should print scan command suggestion", func() { @@ -59,11 +61,11 @@ var _ = Describe("Scan", func() { }) Describe("scan command is run without any flags", func() { - When("scan command is run without any flags, terrascan with scan for terraform files in working directory", func() { + Context("by default, terrascan will scan for terraform files in the working directory", func() { Context("no tf files are present in the working directory", func() { It("should error out as no terraform files are present in working directory", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand) - Eventually(session, 2).Should(gexec.Exit(1)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) helper.ContainsErrorSubString(session, "has no terraform config files") }) }) @@ -72,8 +74,8 @@ var _ = Describe("Scan", func() { It("should scan the directory, return results and exit with status code 3", func() { workDir, err := filepath.Abs("../test_data/iac/aws/aws_ami_violation") Expect(err).NotTo(HaveOccurred()) - session = helper.RunCommandDir(terrascanBinaryPath, workDir, outWriter, errWriter, scanComand) - Eventually(session, 2).Should(gexec.Exit(3)) + session = helper.RunCommandDir(terrascanBinaryPath, workDir, outWriter, errWriter, scanUtils.ScanCommand) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) }) When("tf file present in the dir has no violations", func() { @@ -86,8 +88,8 @@ var _ = Describe("Scan", func() { policyDir, err := filepath.Abs("../test_data/policies/k8s") Expect(err).NotTo(HaveOccurred()) - session = helper.RunCommandDir(terrascanBinaryPath, workDir, outWriter, errWriter, scanComand, "-p", policyDir) - Eventually(session, 2).Should(gexec.Exit(0)) + session = helper.RunCommandDir(terrascanBinaryPath, workDir, outWriter, errWriter, scanUtils.ScanCommand, "-p", policyDir) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeZero)) }) }) }) @@ -95,4 +97,150 @@ var _ = Describe("Scan", func() { }) }) + Describe("terrascan scan command is run with -d and -f flag", func() { + workDir, err := os.Getwd() + It("should not get an error", func() { + Expect(err).NotTo(HaveOccurred()) + }) + Context("invalid path is supplied to -d or -f flag", func() { + + invalidPath := "invalid/path" + When("supplied with -d flag", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", invalidPath) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + + errString := fmt.Sprintf("directory '%s' does not exist", filepath.Join(workDir, invalidPath)) + helper.ContainsErrorSubString(session, errString) + }) + }) + + When("supplied with -f flag", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-f", invalidPath) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + + errString := fmt.Sprintf("file '%s' does not exist", filepath.Join(workDir, invalidPath)) + helper.ContainsErrorSubString(session, errString) + }) + }) + }) + + Context("-d flag is supplied with a valid file path", func() { + It("should error out and exit with status code 1", func() { + validAbsFilePath, err := filepath.Abs("golden/scan_help.txt") + Expect(err).NotTo(HaveOccurred()) + + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", validAbsFilePath) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + + errString := fmt.Sprintf("input path '%s' is not a valid directory", validAbsFilePath) + helper.ContainsErrorSubString(session, errString) + }) + }) + + Context("-f flag is supplied with a valid dir path", func() { + It("should error out and exit with status code 1", func() { + validAbsDirPath, err := filepath.Abs("golden") + Expect(err).NotTo(HaveOccurred()) + + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-f", validAbsDirPath) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + + errString := fmt.Sprintf("input path '%s' is not a valid file", validAbsDirPath) + helper.ContainsErrorSubString(session, errString) + }) + }) + }) + + Describe("scan is run with unsupported iac type or version", func() { + errString := "iac type or version not supported" + When("-i flag is supplied with unsupported iac type", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-i", "test") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + }) + }) + + When("--iac-version flag is supplied invalid version", func() { + Context("default iac type is terraform", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "--iac-version", "test") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + }) + }) + }) + + Context("iac type is valid but iac version isn't", func() { + When("iac type is k8s and supplied version is invalid", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-i", "k8s", "--iac-version", "test") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + + }) + }) + + When("iac type is helm and supplied version is invalid", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-i", "helm", "--iac-version", "test") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + }) + }) + + When("iac type is kustomize and supplied version is invalid", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-i", "kustomize", "--iac-version", "test") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString) + }) + }) + }) + }) + + Describe("scan is run with -p (policy path) flag", func() { + invalidPolicyPath := "path/policy/invalid" + errString1 := "failed to initialize OPA policy engine" + errString2 := fmt.Sprintf("%s: no such file or directory", invalidPolicyPath) + + When("supplied policy path doesn't exist", func() { + It("should error out and exit with status code 1", func() { + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-p", invalidPolicyPath) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString1) + helper.ContainsErrorSubString(session, errString2) + }) + }) + + Context("multiple policy paths can be supplied", func() { + When("one of the supplied policy path is invalid", func() { + It("should error out and exit with staus code 1", func() { + validPolicyPath, err := filepath.Abs("../test_data") + Expect(err).NotTo(HaveOccurred()) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-p", validPolicyPath, "-p", invalidPolicyPath) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + helper.ContainsErrorSubString(session, errString1) + helper.ContainsErrorSubString(session, errString2) + }) + }) + + When("multiple valid policy paths are supplied", func() { + It("should scan with the policies and display results", func() { + validPolicyPath1, err := filepath.Abs("../test_data/policies/aws") + Expect(err).NotTo(HaveOccurred()) + validPolicyPath2, err := filepath.Abs("../test_data/policies/azure") + Expect(err).NotTo(HaveOccurred()) + workDirPath, err := filepath.Abs("../test_data/iac/k8s") + Expect(err).NotTo(HaveOccurred()) + + session = helper.RunCommandDir(terrascanBinaryPath, workDirPath, outWriter, errWriter, scanUtils.ScanCommand, "-p", validPolicyPath1, "-p", validPolicyPath2) + // exits with status code 1, because the work dir has k8s iac file and supplied policies are for tf files + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeOne)) + }) + }) + }) + }) }) diff --git a/test/e2e/scan/scan_tf_files_test.go b/test/e2e/scan/scan_tf_files_test.go index de6481f40..75313f791 100644 --- a/test/e2e/scan/scan_tf_files_test.go +++ b/test/e2e/scan/scan_tf_files_test.go @@ -39,24 +39,24 @@ var _ = Describe("Scan is run for terraform files", func() { When("iac type is not specified, terraform iac provider is used", func() { It("should scan successfully and exit with status code 3", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir) - Eventually(session, 5).Should(gexec.Exit(3)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir) + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) }) }) Context("default iac version for terraform is v14", func() { When("iac version is v12", func() { It("terrascan should display the warning message related to version", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir, "--iac-version", "v12") - Eventually(session, 5).Should(gexec.Exit(3)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir, "--iac-version", "v12") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) helper.ContainsErrorSubString(session, backwardsCompatibilityWarningMessage) }) }) When("iac version is v13", func() { It("terrascan should not display the warning message related to version", func() { - session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanComand, "-d", iacDir, "--iac-version", "v13") - Eventually(session, 5).Should(gexec.Exit(3)) + session = helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, scanUtils.ScanCommand, "-d", iacDir, "--iac-version", "v13") + Eventually(session, scanUtils.ScanTimeout).Should(gexec.Exit(helper.ExitCodeThree)) helper.DoesNotContainsErrorSubString(session, backwardsCompatibilityWarningMessage) }) }) @@ -80,41 +80,41 @@ var _ = Describe("Scan is run for terraform files", func() { Context("iac file violates aws_ami policy", func() { It("should scan and display violations in human output format", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_human.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_human.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) When("-v flag is used for verbose output", func() { It("should display verbose output for human output format", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-v"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_human_verbose.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_human_verbose.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is json", func() { It("should display violations in json format", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_json.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_json.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is yaml", func() { It("should display violations in yaml format", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "yaml"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_yaml.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_yaml.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is xml", func() { It("should display violations in xml format", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "xml"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_xml.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_xml.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is junit-xml", func() { It("should display violations in junit-xml format", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "junit-xml"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_junit_xml.txt", 3, true, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_ami_violations/aws_ami_violation_junit_xml.txt", helper.ExitCodeThree, true, true, outWriter, errWriter, scanArgs...) }) }) @@ -124,7 +124,7 @@ var _ = Describe("Scan is run for terraform files", func() { }) It("should display violations", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json"} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/scanned_with_only_aws_policies.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/scanned_with_only_aws_policies.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) @@ -134,7 +134,7 @@ var _ = Describe("Scan is run for terraform files", func() { }) It("should not display any violations and exit with status code 0", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir} - scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/scanned_with_no_aws_policies.txt", 0, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertTextOutput(terrascanBinaryPath, "golden/terraform_scans/scanned_with_no_aws_policies.txt", helper.ExitCodeZero, false, true, outWriter, errWriter, scanArgs...) }) }) }) @@ -148,21 +148,21 @@ var _ = Describe("Scan is run for terraform files", func() { When("when output type is json", func() { It("should display violations in json format", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "json"} - scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_db_instance_violations/aws_db_instance_json.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertJSONOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_db_instance_violations/aws_db_instance_json.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is yaml", func() { It("should display violations in yaml format", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "yaml"} - scanUtils.RunScanCommandAndAssertYAMLOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_db_instance_violations/aws_db_instance_yaml.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertYAMLOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_db_instance_violations/aws_db_instance_yaml.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) When("when output type is xml", func() { It("should display violations in xml format", func() { scanArgs := []string{"-p", policyDir, "-d", iacDir, "-o", "xml"} - scanUtils.RunScanCommandAndAssertXMLOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_db_instance_violations/aws_db_instance_xml.txt", 3, false, true, outWriter, errWriter, scanArgs...) + scanUtils.RunScanCommandAndAssertXMLOutput(terrascanBinaryPath, "golden/terraform_scans/aws/aws_db_instance_violations/aws_db_instance_xml.txt", helper.ExitCodeThree, false, true, outWriter, errWriter, scanArgs...) }) }) }) diff --git a/test/e2e/scan/scan_utils.go b/test/e2e/scan/scan_utils.go index 9b8c216fa..a28b63469 100644 --- a/test/e2e/scan/scan_utils.go +++ b/test/e2e/scan/scan_utils.go @@ -10,8 +10,10 @@ import ( ) const ( - scanCommand string = "scan" - scanTimeout int = 5 + // ScanCommand is terrascan's scan command + ScanCommand string = "scan" + // ScanTimeout is default scan command execution timeout + ScanTimeout int = 2 ) // RunScanCommandAndAssertTextOutput runs the scan command with supplied paramters and compares actual and golden output @@ -40,10 +42,10 @@ func RunScanCommandAndAssertXMLOutput(terrascanBinaryPath, relGoldenFilePath str // RunScanCommand executes the scan command, validates exit code func RunScanCommand(terrascanBinaryPath, relGoldenFilePath string, exitCode int, outWriter, errWriter io.Writer, args ...string) (*gexec.Session, string) { - argList := []string{scanCommand} + argList := []string{ScanCommand} argList = append(argList, args...) session := helper.RunCommand(terrascanBinaryPath, outWriter, errWriter, argList...) - gomega.Eventually(session, scanTimeout).Should(gexec.Exit(exitCode)) + gomega.Eventually(session, ScanTimeout).Should(gexec.Exit(exitCode)) goldenFileAbsPath, err := filepath.Abs(relGoldenFilePath) gomega.Expect(err).NotTo(gomega.HaveOccurred()) return session, goldenFileAbsPath diff --git a/test/e2e/test_data/iac/resource_skipping/kubernetes/config.yaml b/test/e2e/test_data/iac/resource_skipping/kubernetes/config.yaml new file mode 100644 index 000000000..0d20f3060 --- /dev/null +++ b/test/e2e/test_data/iac/resource_skipping/kubernetes/config.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: production + labels: + "hey": + annotations: + terrascanSkip: + - rule: AC-K8-OE-NS-L-0128 + comment: reason to skip the rule \ No newline at end of file diff --git a/test/e2e/test_data/iac/resource_skipping/terraform/main.tf b/test/e2e/test_data/iac/resource_skipping/terraform/main.tf new file mode 100644 index 000000000..38a4481d7 --- /dev/null +++ b/test/e2e/test_data/iac/resource_skipping/terraform/main.tf @@ -0,0 +1,105 @@ +provider "aws" { + region = "us-west-2" +} + +resource "aws_ebs_volume" "example" { + availability_zone = "us-west-2a" + size = 8 +} + +resource "aws_ebs_snapshot" "example_snapshot" { + volume_id = aws_ebs_volume.example.id +} +resource "aws_ami" "tesami" { + name = "ptshavaa1" + virtualization_type = "hvm" + root_device_name = "/dev/xvda" + + ebs_block_device { + device_name = "/dev/xvda" + snapshot_id = aws_ebs_snapshot.example_snapshot.id + volume_size = 8 + } +} +resource "aws_default_vpc" "main" { + tags = { + Name = "Default VPC" + } +} + +resource "aws_security_group" "allow_tls" { + name = "ptshavasg1" + description = "Allow TLS inbound traffic" + vpc_id = aws_default_vpc.main.id + + ingress { + description = "TLS from VPC" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = [aws_default_vpc.main.cidr_block] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_instance" "instanceWithNoVpc" { + ami = aws_ami.tesami.id + instance_type = "t2.micro" + + metadata_options { + http_endpoint = "disabled" + http_tokens = "required" + } +} + +resource "aws_instance" "instanceWithPublicIp" { + #ts:skip=AC-AWS-NS-IN-M-1172 should skip this rule + ami = aws_ami.tesami.id + instance_type = "t2.micro" + hibernation = false + + associate_public_ip_address = true +} + +resource "aws_instance" "instanceWithIMDv1_emptyblock" { + ami = "ami-1234" + instance_type = "t2.micro" + + metadata_options {} +} + +resource "aws_instance" "instanceWithIMDv1_fullblock" { + ami = "ami-1234" + instance_type = "t2.micro" + + metadata_options { + http_endpoint = "enabled" + http_tokens = "optional" + } +} + +resource "aws_instance" "instanceWithIMDv1_token_not_present" { + #ts:skip=AC-AWS-NS-IN-M-1172 need to skip this rule + #ts:skip=AC-AW-IS-IN-M-0144 can skip this rule + ami = "ami-1234" + instance_type = "t2.micro" + + metadata_options { + http_endpoint = "enabled" + } +} + +resource "aws_instance" "instanceWithIMDv1_endpoint_not_present" { + ami = "ami-1234" + instance_type = "t2.micro" + + metadata_options { + http_tokens = "optional" + } +} \ No newline at end of file diff --git a/test/helper/helper.go b/test/helper/helper.go index 164617ebd..8583e6972 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -21,6 +21,15 @@ import ( "gopkg.in/yaml.v3" ) +const ( + // ExitCodeZero represents command exit code 0 + ExitCodeZero = 0 + // ExitCodeOne represents command exit code 0 + ExitCodeOne = 1 + // ExitCodeThree represents command exit code 0 + ExitCodeThree = 3 +) + var ( // ScannedAt is regex for 'scanned at' attribute in violations output ScannedAt = regexp.MustCompile(`["]*[sS]canned[ _][aA]t["]*[ \t]*[:=][ \t]*["]*[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}[.][0-9]{1,9} [+-][0-9]{4} UTC["]*[,]{0,1}`) @@ -203,6 +212,7 @@ func GetByteData(session *gexec.Session, goldenFileAbsPath string, isStdOut bool func compareSummaryAndViolations(sessionEngineOutput, fileDataEngineOutput policy.EngineOutput) { var sessionOutputSummary, fileDataSummary results.ScanSummary var actualViolations, expectedViolations violations + var actualSkippedViolations, expectedSkippedViolations violations actualViolations = sessionEngineOutput.ViolationStore.Violations expectedViolations = fileDataEngineOutput.ViolationStore.Violations @@ -210,6 +220,12 @@ func compareSummaryAndViolations(sessionEngineOutput, fileDataEngineOutput polic sort.Sort(actualViolations) sort.Sort(expectedViolations) + actualSkippedViolations = sessionEngineOutput.ViolationStore.SkippedViolations + expectedSkippedViolations = fileDataEngineOutput.ViolationStore.SkippedViolations + + sort.Sort(actualSkippedViolations) + sort.Sort(expectedSkippedViolations) + sessionOutputSummary = sessionEngineOutput.ViolationStore.Summary removeTimestampAndResourcePath(&sessionOutputSummary) @@ -218,6 +234,7 @@ func compareSummaryAndViolations(sessionEngineOutput, fileDataEngineOutput polic gomega.Expect(reflect.DeepEqual(sessionOutputSummary, fileDataSummary)).To(gomega.BeTrue()) gomega.Expect(reflect.DeepEqual(actualViolations, expectedViolations)).To(gomega.BeTrue()) + gomega.Expect(reflect.DeepEqual(actualSkippedViolations, expectedSkippedViolations)).To(gomega.BeTrue()) } // removeTimestampAndResourcePath is helper func to make timestamp and resource path blank