Skip to content

Commit

Permalink
Revamp tests for v0.2.2 add more fixtures
Browse files Browse the repository at this point in the history
This commit udpates and improves the project's tests to
work with the new spec v0.2.0. It also adds various new
documents to widen the test cases and ensure compatibility
with the previous spec version.

Signed-off-by: Adolfo García Veytia (Puerco) <[email protected]>
  • Loading branch information
puerco committed Aug 24, 2023
1 parent 30bb90a commit 4a435bb
Show file tree
Hide file tree
Showing 14 changed files with 12,840 additions and 74 deletions.
101 changes: 35 additions & 66 deletions pkg/ctl/ctl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ SPDX-License-Identifier: Apache-2.0
package ctl

import (
"context"
"testing"

"github.com/stretchr/testify/require"
Expand All @@ -16,74 +15,44 @@ import (
)

func TestVexReport(t *testing.T) {
vexDoc, err := vex.OpenJSON("testdata/test.vex.json")
require.NoError(t, err)
require.NotNil(t, vexDoc)
require.Len(t, vexDoc.Statements, 2)

report, err := sarif.Open("testdata/nginx.sarif.json")
require.NoError(t, err)
require.NotNil(t, report)
require.Len(t, report.Runs, 1)
require.Len(t, report.Runs[0].Results, 123)

impl := defaultVexCtlImplementation{}
newReport, err := impl.ApplySingleVEX(report, vexDoc)
require.NoError(t, err)
require.Len(t, newReport.Runs, 1)
require.Len(t, newReport.Runs[0].Results, 122)
}

func TestMerge(t *testing.T) {
ctx := context.Background()
doc1, err := vex.Load("testdata/document1.vex.json")
require.NoError(t, err)
doc2, err := vex.Load("testdata/document1.vex.json")
require.NoError(t, err)

impl := defaultVexCtlImplementation{}
for _, tc := range []struct {
opts MergeOptions
docs []*vex.VEX
expectedDoc *vex.VEX
shouldErr bool
vexDoc string
lenStatements int
sarifDoc string
lenRuns int
lenResults int
lenAfterFilter int
}{
// Zero docs should fail
{
opts: MergeOptions{},
docs: []*vex.VEX{},
expectedDoc: &vex.VEX{},
shouldErr: true,
},
// One doc results in the same doc
{
opts: MergeOptions{},
docs: []*vex.VEX{doc1},
expectedDoc: doc1,
shouldErr: false,
},
// Two docs, as they are
{
opts: MergeOptions{},
docs: []*vex.VEX{doc1, doc2},
expectedDoc: &vex.VEX{
Metadata: vex.Metadata{},
Statements: []vex.Statement{
doc1.Statements[0],
doc2.Statements[0],
},
},
shouldErr: false,
},
// One OpenVEX statement, filters one vuln
{"testdata/sarif/sample.openvex.json", 1, "testdata/sarif/nginx-grype.sarif.json", 1, 99, 98},
{"testdata/sarif/sample.openvex.json", 1, "testdata/sarif/nginx-trivy.sarif.json", 1, 99, 98},
{"testdata/sarif/sample.openvex.json", 1, "testdata/sarif/nginx-snyk.sarif.json", 2, 65, 64},

// Two OpenVEX statements, filters one vuln
{"testdata/sarif/sample-history.json", 2, "testdata/sarif/nginx-grype.sarif.json", 1, 99, 98},
{"testdata/sarif/sample-history.json", 2, "testdata/sarif/nginx-trivy.sarif.json", 1, 99, 98},
{"testdata/sarif/sample-history.json", 2, "testdata/sarif/nginx-snyk.sarif.json", 2, 65, 64},

// Two OpenVEX statements, filters two vuln
{"testdata/sarif/sample-2vulns.json", 2, "testdata/sarif/nginx-grype.sarif.json", 1, 99, 96},
{"testdata/sarif/sample-2vulns.json", 2, "testdata/sarif/nginx-trivy.sarif.json", 1, 99, 96},
{"testdata/sarif/sample-2vulns.json", 2, "testdata/sarif/nginx-snyk.sarif.json", 2, 65, 63},
} {
doc, err := impl.Merge(ctx, &tc.opts, tc.docs)
if tc.shouldErr {
require.Error(t, err)
continue
}

// Check doc
require.Len(t, doc.Statements, len(tc.expectedDoc.Statements))
require.Equal(t, doc.Statements, tc.expectedDoc.Statements)
vexDoc, err := vex.Open(tc.vexDoc)
require.NoError(t, err)
require.NotNil(t, vexDoc)
require.Len(t, vexDoc.Statements, tc.lenStatements)

report, err := sarif.Open(tc.sarifDoc)
require.NoError(t, err)
require.NotNil(t, report)
require.Len(t, report.Runs, tc.lenRuns)
require.Len(t, report.Runs[0].Results, tc.lenResults)

newReport, err := impl.ApplySingleVEX(report, vexDoc)
require.NoError(t, err)
require.Len(t, newReport.Runs, tc.lenRuns)
require.Len(t, newReport.Runs[0].Results, tc.lenAfterFilter)
}
}
109 changes: 106 additions & 3 deletions pkg/ctl/implementation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ SPDX-License-Identifier: Apache-2.0
package ctl

import (
"context"
"testing"

intoto "github.com/in-toto/in-toto-golang/in_toto"
Expand Down Expand Up @@ -106,11 +107,15 @@ func TestListDocumentProducts(t *testing.T) {
},
},
{
"testdata/document1.vex.json",
"testdata/v001-1.vex.json",
[]string{"pkg:apk/wolfi/[email protected]"},
},
{
"testdata/v020-1.vex.json",
[]string{"pkg:apk/wolfi/[email protected]"},
},
} {
doc, err := vex.OpenJSON(tc.path)
doc, err := vex.Open(tc.path)
require.NoError(t, err)
prods, err := impl.ListDocumentProducts(doc)
require.NoError(t, err)
Expand Down Expand Up @@ -155,7 +160,18 @@ func TestVerifyImageSubjects(t *testing.T) {
doc := vex.New()
for _, p := range tc.products {
doc.Statements = append(
doc.Statements, vex.Statement{Products: []string{p}},
doc.Statements, vex.Statement{
Products: []vex.Product{
{
Component: vex.Component{
ID: p,
Hashes: map[vex.Algorithm]vex.Hash{},
Identifiers: map[vex.IdentifierType]string{},
},
Subcomponents: []vex.Subcomponent{},
},
},
},
)
}
err := impl.VerifyImageSubjects(att, &doc)
Expand All @@ -166,3 +182,90 @@ func TestVerifyImageSubjects(t *testing.T) {
}
}
}

func TestMerge(t *testing.T) {
ctx := context.Background()
doc1, err := vex.Open("testdata/v001-1.vex.json")
require.NoError(t, err)
doc2, err := vex.Open("testdata/v001-2.vex.json")
require.NoError(t, err)

doc3, err := vex.Open("testdata/v020-1.vex.json")
require.NoError(t, err)
doc4, err := vex.Open("testdata/v020-2.vex.json")
require.NoError(t, err)

impl := defaultVexCtlImplementation{}
for _, tc := range []struct {
opts MergeOptions
docs []*vex.VEX
expectedDoc *vex.VEX
shouldErr bool
}{
// Zero docs should fail
{
opts: MergeOptions{},
docs: []*vex.VEX{},
expectedDoc: &vex.VEX{},
shouldErr: true,
},
// One doc results in the same doc
{
opts: MergeOptions{},
docs: []*vex.VEX{doc1},
expectedDoc: doc1,
shouldErr: false,
},
// Two docs, as they are
{
opts: MergeOptions{},
docs: []*vex.VEX{doc1, doc2},
expectedDoc: &vex.VEX{
Metadata: vex.Metadata{},
Statements: []vex.Statement{
doc1.Statements[0],
doc2.Statements[0],
},
},
shouldErr: false,
},
// Two docs, filter product
{
opts: MergeOptions{
Products: []string{"pkg:apk/wolfi/[email protected]"},
},
docs: []*vex.VEX{doc3, doc4},
expectedDoc: &vex.VEX{
Metadata: vex.Metadata{},
Statements: []vex.Statement{
doc4.Statements[0],
},
},
shouldErr: false,
},
// Two docs, filter vulnerability
{
opts: MergeOptions{
Vulnerabilities: []string{"CVE-9876-54321"},
},
docs: []*vex.VEX{doc3, doc4},
expectedDoc: &vex.VEX{
Metadata: vex.Metadata{},
Statements: []vex.Statement{
doc3.Statements[0],
},
},
shouldErr: false,
},
} {
doc, err := impl.Merge(ctx, &tc.opts, tc.docs)
if tc.shouldErr {
require.Error(t, err)
continue
}

// Check doc
require.Len(t, doc.Statements, len(tc.expectedDoc.Statements))
require.Equal(t, doc.Statements, tc.expectedDoc.Statements)
}
}
4 changes: 2 additions & 2 deletions pkg/ctl/testdata/images.vex.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"id": "my-vexdoc",
"format": "text/vex+json",
"@context": "https://openvex.dev/ns",
"@id": "https://openvex.dev/docs/public/vex-c387e5e30d92b7a54412468ebc3af98805b5d8a8e60e9b1cabda755ff6a6ae57",
"author": "John Doe",
"role": "vex issuer",
"statements": [
Expand Down
Loading

0 comments on commit 4a435bb

Please sign in to comment.