From 0be1ef0d37e1e9ec7ecdd884410c609a8f23193c Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Wed, 27 Dec 2023 20:50:03 +0000 Subject: [PATCH 1/2] fix: improve vuln report messages --- .../vulnerability_report.go | 26 ++++++++++--------- .../vulnerability_report_test.go | 10 +++---- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report.go b/plugins/verifier/vulnerabilityreport/vulnerability_report.go index a3eb495c0..2728e5f95 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report.go @@ -136,7 +136,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("error fetching reference manifest for subject: %s reference descriptor: %v: [%v]", subjectReference, referenceDescriptor.Descriptor, err.Error()), + Message: fmt.Sprintf("vulnerability report validation failed: error fetching reference manifest for subject: %s reference descriptor: %v: [%v]", subjectReference, referenceDescriptor.Descriptor, err.Error()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -162,7 +162,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("error fetching blob for subject:[%s] digest:[%s]: [%v]", subjectReference, blobDesc.Digest, err.Error()), + Message: fmt.Sprintf("vulnerability report validation failed: error fetching blob for subject:[%s] digest:[%s]: [%v]", subjectReference, blobDesc.Digest, err.Error()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -175,7 +175,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: true, - Message: "vulnerability report validation skipped", + Message: "vulnerability report validation skipped. passthrough enabled", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, "passthrough": true, @@ -332,10 +332,11 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st IsSuccess: false, Extensions: map[string]interface{}{ "scanner": scannerName, - "denylistCVEs": denylistViolations, + "denylistCVEs": denylistCVEs, + "violatingCVEs": denylistViolations, CreatedAnnotation: createdTime, }, - Message: "vulnerability report validation failed", + Message: "vulnerability report validation failed: deny listed CVEs found. please review extensions field", }, nil } @@ -354,7 +355,7 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st // verifyDisallowedSeverities verifies that the report does not contain any disallowed severity levels func verifyDisallowedSeverities(verifierName string, verifierType string, scannerName string, sarifReport *sarif.Report, disallowedSeverities []string, createdTime time.Time) (*verifier.VerifierResult, error) { ruleMap := make(map[string]*sarif.ReportingDescriptor) - violatingRules := make([]sarif.ReportingDescriptor, 0) + violatingRules := make(map[string]string) // create a map of rule id to rule for easy lookup for _, rule := range sarifReport.Runs[0].Tool.Driver.Rules { ruleMap[rule.ID] = rule @@ -399,10 +400,10 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne }, }, nil } - // check if the severity is disallowed and add it to the list of violating rules + // check if the severity is disallowed and add it to the map of violating CVE IDs for _, disallowed := range disallowedSeverities { if strings.EqualFold(severity, disallowed) { - violatingRules = append(violatingRules, *rule) + violatingRules[rule.ID] = severity } } } @@ -413,11 +414,12 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne Type: verifierType, IsSuccess: false, Extensions: map[string]interface{}{ - "scanner": scannerName, - "severityViolations": violatingRules, - CreatedAnnotation: createdTime, + "scanner": scannerName, + "disallowedSeverities": disallowedSeverities, + "severityViolations": violatingRules, + CreatedAnnotation: createdTime, }, - Message: "vulnerability report validation failed", + Message: "vulnerability report validation failed: disallowed severities found. please review extensions field", }, nil } return &verifier.VerifierResult{ diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go index 1e2adff47..908d7f207 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go @@ -169,7 +169,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "vulnerability report validation skipped", + message: "vulnerability report validation skipped. passthrough enabled", }, }, { @@ -357,7 +357,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "vulnerability report validation failed", + message: "vulnerability report validation failed: deny listed CVEs found. please review extensions field", err: nil, }, }, @@ -374,7 +374,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "vulnerability report validation failed", + message: "vulnerability report validation failed: disallowed severities found. please review extensions field", err: nil, }, }, @@ -483,7 +483,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: "vulnerability report validation failed", + message: "vulnerability report validation failed: deny listed CVEs found. please review extensions field", err: nil, }, }, @@ -687,7 +687,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: "vulnerability report validation failed", + message: "vulnerability report validation failed: disallowed severities found. please review extensions field", err: nil, }, }, From 29c375c25187e0c4e9867dd0f2be0421cc2a91fc Mon Sep 17 00:00:00 2001 From: Akash Singhal Date: Thu, 4 Jan 2024 19:20:06 +0000 Subject: [PATCH 2/2] update error messages --- .../vulnerability_report.go | 42 +++++++++--------- .../vulnerability_report_test.go | 44 +++++++++---------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report.go b/plugins/verifier/vulnerabilityreport/vulnerability_report.go index 2728e5f95..e6820ac3b 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report.go @@ -97,7 +97,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: error extracting create timestamp annotation:[%v]", err.Error()), + Message: fmt.Sprintf("Validation failed: error extracting create timestamp annotation:[%v]", err.Error()), }, nil } @@ -109,7 +109,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: error validating maximum age:[%v]", err.Error()), + Message: fmt.Sprintf("Validation failed: error validating maximum age:[%v]", err.Error()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -120,7 +120,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: report is older than maximum age:[%s]", input.MaximumAge), + Message: fmt.Sprintf("Validation failed: report is older than maximum age:[%s]", input.MaximumAge), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -136,7 +136,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: error fetching reference manifest for subject: %s reference descriptor: %v: [%v]", subjectReference, referenceDescriptor.Descriptor, err.Error()), + Message: fmt.Sprintf("Validation failed: error fetching reference manifest for subject: %s reference descriptor: %v: [%v]", subjectReference, referenceDescriptor.Descriptor, err.Error()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -148,7 +148,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: no layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), + Message: fmt.Sprintf("Validation failed: no layers found in manifest for referrer %s@%s", subjectReference.Path, referenceDescriptor.Digest.String()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -162,7 +162,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: error fetching blob for subject:[%s] digest:[%s]: [%v]", subjectReference, blobDesc.Digest, err.Error()), + Message: fmt.Sprintf("Validation failed: error fetching blob for subject:[%s] digest:[%s]: [%v]", subjectReference, blobDesc.Digest, err.Error()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -175,7 +175,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: true, - Message: "vulnerability report validation skipped. passthrough enabled", + Message: "Validation skipped. passthrough enabled", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, "passthrough": true, @@ -190,7 +190,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDesc.Digest, referenceDescriptor.ArtifactType, err.Error()), + Message: fmt.Sprintf("Validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDesc.Digest, referenceDescriptor.ArtifactType, err.Error()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -205,7 +205,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe Name: input.Name, Type: verifierType, IsSuccess: true, - Message: "vulnerability report validation succeeded", + Message: "Validation succeeded", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -238,7 +238,7 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s Name: verifierName, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: error parsing sarif report:[%v]", err.Error()), + Message: fmt.Sprintf("Validation failed: error parsing sarif report:[%v]", err.Error()), Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -250,7 +250,7 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s Name: verifierName, Type: verifierType, IsSuccess: false, - Message: "vulnerability report validation failed: no runs found in sarif report", + Message: "Validation failed: no runs found in sarif report", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, }, @@ -280,7 +280,7 @@ func processSarifReport(input *PluginConfig, verifierName string, verifierType s Name: verifierName, Type: verifierType, IsSuccess: true, - Message: "vulnerability report validation succeeded", + Message: "Validation succeeded", Extensions: map[string]interface{}{ CreatedAnnotation: createdTime, "scanner": scannerName, @@ -305,7 +305,7 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st Name: verifierName, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: rule id not found for result:[%v]", result), + Message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", result), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -333,10 +333,10 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st Extensions: map[string]interface{}{ "scanner": scannerName, "denylistCVEs": denylistCVEs, - "violatingCVEs": denylistViolations, + "cveViolations": denylistViolations, CreatedAnnotation: createdTime, }, - Message: "vulnerability report validation failed: deny listed CVEs found. please review extensions field", + Message: "Validation failed: found denied CVEs. See extensions field for details.", }, nil } @@ -344,7 +344,7 @@ func verifyDenyListCVEs(verifierName string, verifierType string, scannerName st Name: verifierName, Type: verifierType, IsSuccess: true, - Message: "vulnerability report validation succeeded", + Message: "Validation succeeded", Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -367,7 +367,7 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne Name: verifierName, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: rule id not found for result:[%v]", result), + Message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", result), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -380,7 +380,7 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne Name: verifierName, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: rule not found for result:[%v]", result), + Message: fmt.Sprintf("Validation failed: rule not found for result:[%v]", result), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -393,7 +393,7 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne Name: verifierName, Type: verifierType, IsSuccess: false, - Message: fmt.Sprintf("vulnerability report validation failed: error extracting severity:[%v]", err.Error()), + Message: fmt.Sprintf("Validation failed: error extracting severity:[%v]", err.Error()), Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, @@ -419,14 +419,14 @@ func verifyDisallowedSeverities(verifierName string, verifierType string, scanne "severityViolations": violatingRules, CreatedAnnotation: createdTime, }, - Message: "vulnerability report validation failed: disallowed severities found. please review extensions field", + Message: "Validation failed: found disallowed severities. See extensions field for details.", }, nil } return &verifier.VerifierResult{ Name: verifierName, Type: verifierType, IsSuccess: true, - Message: "vulnerability report validation succeeded", + Message: "Validation succeeded", Extensions: map[string]interface{}{ "scanner": scannerName, CreatedAnnotation: createdTime, diff --git a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go index 908d7f207..9710746af 100644 --- a/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go +++ b/plugins/verifier/vulnerabilityreport/vulnerability_report_test.go @@ -103,7 +103,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: error extracting create timestamp annotation:[%s]", "no annotations found for descriptor:[{{ sha256:b2f67b016d3c646f025099b363b4f83a56a44d067a846be74e8866342c56f216 0 [] map[] [] } application/sarif+json}]"), + message: fmt.Sprintf("Validation failed: error extracting create timestamp annotation:[%s]", "no annotations found for descriptor:[{{ sha256:b2f67b016d3c646f025099b363b4f83a56a44d067a846be74e8866342c56f216 0 [] map[] [] } application/sarif+json}]"), }, }, { @@ -118,7 +118,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: error validating maximum age:[%s]", "error parsing maximum age:[1d]"), + message: fmt.Sprintf("Validation failed: error validating maximum age:[%s]", "error parsing maximum age:[1d]"), }, }, { @@ -133,7 +133,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: report is older than maximum age:[%s]", "24h"), + message: fmt.Sprintf("Validation failed: report is older than maximum age:[%s]", "24h"), }, }, { @@ -148,7 +148,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: no layers found in manifest for referrer %s@%s", "test_subject_path", manifestDigest.String()), + message: fmt.Sprintf("Validation failed: no layers found in manifest for referrer %s@%s", "test_subject_path", manifestDigest.String()), }, }, { @@ -169,7 +169,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "vulnerability report validation skipped. passthrough enabled", + message: "Validation skipped. passthrough enabled", }, }, { @@ -189,7 +189,7 @@ func TestVerifyReference(t *testing.T) { blobContent: "{}", }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDigest, SarifArtifactType, "version is required: runs is required: "), + message: fmt.Sprintf("Validation failed: schema validation failed for digest:[%s],artifact type:[%s],parse errors:[%v]", blobDigest, SarifArtifactType, "version is required: runs is required: "), }, }, { @@ -210,7 +210,7 @@ func TestVerifyReference(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "vulnerability report validation succeeded", + message: "Validation succeeded", }, }, } @@ -328,7 +328,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: "invalid", }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: error parsing sarif report:[%s]", "invalid character 'i' looking for beginning of value"), + message: fmt.Sprintf("Validation failed: error parsing sarif report:[%s]", "invalid character 'i' looking for beginning of value"), err: nil, }, }, @@ -343,7 +343,7 @@ func TestProcessSarifReport(t *testing.T) { }`, }, want: want{ - message: "vulnerability report validation failed: no runs found in sarif report", + message: "Validation failed: no runs found in sarif report", err: nil, }, }, @@ -357,7 +357,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "vulnerability report validation failed: deny listed CVEs found. please review extensions field", + message: "Validation failed: found denied CVEs. See extensions field for details.", err: nil, }, }, @@ -374,12 +374,12 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "vulnerability report validation failed: disallowed severities found. please review extensions field", + message: "Validation failed: found disallowed severities. See extensions field for details.", err: nil, }, }, { - name: "vulnerability report validation succeeded", + name: "Validation succeeded", args: args{ input: PluginConfig{ Name: "test_verifier", @@ -391,7 +391,7 @@ func TestProcessSarifReport(t *testing.T) { blobContent: sampleSarifReport, }, want: want{ - message: "vulnerability report validation succeeded", + message: "Validation succeeded", err: nil, }, }, @@ -452,7 +452,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: rule id not found for result:[%v]", &sarif.Result{}), + message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", &sarif.Result{}), err: nil, }, }, @@ -483,7 +483,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: "vulnerability report validation failed: deny listed CVEs found. please review extensions field", + message: "Validation failed: found denied CVEs. See extensions field for details.", err: nil, }, }, @@ -514,7 +514,7 @@ func TestVerifyDenyListCVEs(t *testing.T) { }, }, want: want{ - message: "vulnerability report validation succeeded", + message: "Validation succeeded", err: nil, }, }, @@ -582,7 +582,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: rule id not found for result:[%v]", &sarif.Result{}), + message: fmt.Sprintf("Validation failed: rule id not found for result:[%v]", &sarif.Result{}), err: nil, }, }, @@ -617,7 +617,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: rule not found for result:[%v]", &sarif.Result{RuleID: &invalidRuleID}), + message: fmt.Sprintf("Validation failed: rule not found for result:[%v]", &sarif.Result{RuleID: &invalidRuleID}), err: nil, }, }, @@ -652,7 +652,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: fmt.Sprintf("vulnerability report validation failed: error extracting severity:[severity not found in help text:[%s]]", invalidSeverityText), + message: fmt.Sprintf("Validation failed: error extracting severity:[severity not found in help text:[%s]]", invalidSeverityText), err: nil, }, }, @@ -687,12 +687,12 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: "vulnerability report validation failed: disallowed severities found. please review extensions field", + message: "Validation failed: found disallowed severities. See extensions field for details.", err: nil, }, }, { - name: "vulnerability report validation succeeded", + name: "Validation succeeded", args: args{ disallowedSeverities: []string{"critical"}, sarifReport: sarif.Report{ @@ -722,7 +722,7 @@ func TestVerifyDisallowedSeverities(t *testing.T) { }, }, want: want{ - message: "vulnerability report validation succeeded", + message: "Validation succeeded", err: nil, }, },