diff --git a/coverage/coverage.go b/coverage/coverage.go index a8de86bb..b5f35420 100644 --- a/coverage/coverage.go +++ b/coverage/coverage.go @@ -17,31 +17,23 @@ package coverage import ( + "bufio" + "bytes" "fmt" "golang.org/x/tools/cover" "io" "os" "os/exec" + "strings" ) -// Block holds the start and end coordinates of a section of a source file -// covered by tests. -type Block struct { - StartLine int - StartCol int - EndLine int - EndCol int -} - -// Profile is implemented as a map holding a slice of Block per each filename. -type Profile map[string][]Block - // Coverage is responsible for executing a Go test with coverage via the Run() method, // then parsing the result coverage report file. type Coverage struct { cmdContext execContext workDir string fileName string + mod string } type execContext = func(name string, args ...string) *exec.Cmd @@ -49,22 +41,38 @@ type execContext = func(name string, args ...string) *exec.Cmd // New instantiates a Coverage element using exec.Command as execContext, // actually running the command on the OS. func New(workdir string) *Coverage { - return NewWithCmd(exec.Command, workdir) + return NewWithCmdAndPackage(exec.Command, getMod(), workdir) +} +func getMod() string { + file, err := os.Open("go.mod") + if err != nil { + fmt.Printf("not in a go module root folder: %v\n", err) + os.Exit(-1) + } + r := bufio.NewReader(file) + line, _, err := r.ReadLine() + if err != nil { + fmt.Printf("not in a go module root folder %v\n\n", err) + os.Exit(-1) + } + packageName := bytes.TrimPrefix(line, []byte("module ")) + return string(packageName) } -// NewWithCmd instantiates a Coverage element given a custom execContext. -func NewWithCmd(cmdContext execContext, workdir string) *Coverage { +// NewWithCmdAndPackage instantiates a Coverage element given a custom execContext. +func NewWithCmdAndPackage(cmdContext execContext, mod, workdir string) *Coverage { return &Coverage{ cmdContext: cmdContext, workDir: workdir, fileName: "coverage", + mod: mod, } } // Run executes the coverage command and parses the results, returning a *Profile // object. -func (c Coverage) Run() (*Profile, error) { +func (c Coverage) Run() (Profile, error) { err := c.execute() if err != nil { return nil, err @@ -77,12 +85,12 @@ func (c Coverage) Run() (*Profile, error) { return profile, nil } -func (c Coverage) getProfile(err error) (*Profile, error) { +func (c Coverage) getProfile(err error) (Profile, error) { cf, err := os.Open(c.filePath()) if err != nil { return nil, err } - profile, err := parse(cf) + profile, err := c.parse(cf) if err != nil { return nil, err } @@ -103,7 +111,7 @@ func (c Coverage) execute() error { return nil } -func parse(data io.Reader) (*Profile, error) { +func (c Coverage) parse(data io.Reader) (Profile, error) { profiles, err := cover.ParseProfilesFromReader(data) if err != nil { return nil, err @@ -117,8 +125,13 @@ func parse(data io.Reader) (*Profile, error) { EndLine: b.EndLine, EndCol: b.EndCol, } - status[p.FileName] = append(status[p.FileName], block) + fn := removeModuleFromPath(p, c) + status[fn] = append(status[fn], block) } } - return &status, nil + return status, nil +} + +func removeModuleFromPath(p *cover.Profile, c Coverage) string { + return strings.ReplaceAll(p.FileName, c.mod+"/", "") } diff --git a/coverage/coverage_test.go b/coverage/coverage_test.go index 0ac5526c..e65b33e6 100644 --- a/coverage/coverage_test.go +++ b/coverage/coverage_test.go @@ -17,10 +17,12 @@ package coverage_test import ( + "fmt" "github.com/google/go-cmp/cmp" "github.com/k3rn31/gremlins/coverage" "os" "os/exec" + "strings" "testing" ) @@ -34,34 +36,22 @@ func TestCoverageRun(t *testing.T) { wantWorkdir := "workdir" wantFilename := "coverage" wantFilePath := wantWorkdir + "/" + wantFilename - got := &commandHolder{} - cov := coverage.NewWithCmd(fakeExecCommandSuccess(got), wantWorkdir) + holder := &commandHolder{} + cov := coverage.NewWithCmdAndPackage(fakeExecCommandSuccess(holder), "example.com", wantWorkdir) _, _ = cov.Run() - if !cmp.Equal(got.command, "go") { - t.Errorf("expected commandHolder to be 'go', got %q", got.command) - } - if !cmp.Equal(got.args[0], "test") { - t.Errorf("expected 'test', got %q", got.args[0]) - } - if !cmp.Equal(got.args[1], "-cover") { - t.Errorf("expected '-cover', got %q", got.args[0]) - } - if !cmp.Equal(got.args[2], "-coverprofile") { - t.Errorf("expected '-coverprofile', got %q", got.args[0]) - } - if !cmp.Equal(got.args[3], wantFilePath) { - t.Errorf("expected %q, got %q", wantFilePath, got.args[3]) - } - if !cmp.Equal(got.args[4], "./...") { - t.Errorf("expected %q, got %q", "./...", got.args[4]) + want := fmt.Sprintf("go test -cover -coverprofile %v ./...", wantFilePath) + got := fmt.Sprintf("go %v", strings.Join(holder.args, " ")) + + if !cmp.Equal(got, want) { + t.Errorf(cmp.Diff(got, want)) } } func TestCoverageRunFails(t *testing.T) { t.Parallel() - cov := coverage.NewWithCmd(fakeExecCommandFailure, "workdir") + cov := coverage.NewWithCmdAndPackage(fakeExecCommandFailure, "example.com", "workdir") _, err := cov.Run() if err == nil { t.Error("expected run to report an error") @@ -70,9 +60,9 @@ func TestCoverageRunFails(t *testing.T) { func TestCoverageParsesOutput(t *testing.T) { t.Parallel() - cov := coverage.NewWithCmd(fakeExecCommandSuccess(nil), "testdata/valid") - want := &coverage.Profile{ - "example.com/path/file1.go": { + cov := coverage.NewWithCmdAndPackage(fakeExecCommandSuccess(nil), "example.com", "testdata/valid") + want := coverage.Profile{ + "path/file1.go": { { StartLine: 47, StartCol: 2, @@ -86,7 +76,7 @@ func TestCoverageParsesOutput(t *testing.T) { EndCol: 20, }, }, - "example.com/path2/file2.go": { + "path2/file2.go": { { StartLine: 52, StartCol: 2, @@ -108,7 +98,7 @@ func TestCoverageParsesOutput(t *testing.T) { func TestParseOutputFail(t *testing.T) { t.Parallel() - cov := coverage.NewWithCmd(fakeExecCommandSuccess(nil), "testdata/invalid") + cov := coverage.NewWithCmdAndPackage(fakeExecCommandSuccess(nil), "example.com", "testdata/invalid") _, err := cov.Run() if err == nil { diff --git a/coverage/profile.go b/coverage/profile.go new file mode 100644 index 00000000..e115a235 --- /dev/null +++ b/coverage/profile.go @@ -0,0 +1,48 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coverage + +import ( + "go/token" +) + +// Block holds the start and end coordinates of a section of a source file +// covered by tests. +type Block struct { + StartLine int + StartCol int + EndLine int + EndCol int +} + +// Profile is implemented as a map holding a slice of Block per each filename. +type Profile map[string][]Block + +func (p Profile) IsCovered(pos token.Position) bool { + block, ok := p[pos.Filename] + if !ok { + return false + } + for _, b := range block { + if pos.Line >= b.StartLine && pos.Line <= b.EndLine { + if pos.Column >= b.StartCol && pos.Column <= b.EndCol { + return true + } + } + } + return false +} diff --git a/coverage/profile_test.go b/coverage/profile_test.go new file mode 100644 index 00000000..a4332a1c --- /dev/null +++ b/coverage/profile_test.go @@ -0,0 +1,118 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coverage_test + +import ( + "github.com/k3rn31/gremlins/coverage" + "go/token" + "testing" +) + +func TestIsCovered(t *testing.T) { + testCases := []struct { + name string + proFilename string + proStartL int + proEndL int + proStartC int + proEndC int + + posFilename string + posL int + posC int + + expected bool + }{ + { + name: "returns true when line and column are covered", + proFilename: "test", + proStartL: 10, + proEndL: 10, + proStartC: 10, + proEndC: 10, + posFilename: "test", + posL: 10, + posC: 10, + expected: true, + }, + { + name: "returns false when line is not covered and column is covered", + proFilename: "test", + proStartL: 10, + proEndL: 10, + proStartC: 10, + proEndC: 10, + posFilename: "test", + posL: 11, + posC: 10, + expected: false, + }, + { + name: "returns false when line is covered and column not covered", + proFilename: "test", + proStartL: 10, + proEndL: 10, + proStartC: 10, + proEndC: 10, + posFilename: "test", + posL: 10, + posC: 11, + expected: false, + }, + { + name: "returns false when filename is not found", + proFilename: "test_pro", + proStartL: 10, + proEndL: 10, + proStartC: 10, + proEndC: 10, + posFilename: "test_pos", + posL: 10, + posC: 10, + expected: false, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + profile := coverage.Profile{ + tc.proFilename: { + { + StartLine: tc.proStartL, + StartCol: tc.proStartC, + EndLine: tc.proEndL, + EndCol: tc.proEndC, + }, + }, + } + + position := token.Position{ + Filename: tc.posFilename, + Offset: 100, + Line: tc.posL, + Column: tc.posC, + } + + got := profile.IsCovered(position) + + if got != tc.expected { + t.Errorf("expected coverage to be %v, got %v", tc.expected, got) + } + }) + } +} diff --git a/mutator/mappings.go b/mutator/mappings.go new file mode 100644 index 00000000..cd20dc8e --- /dev/null +++ b/mutator/mappings.go @@ -0,0 +1,50 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mutator + +import "go/token" + +type MutantType int + +func (mt MutantType) String() string { + switch mt { + case ConditionalBoundary: + return "Conditional Boundary" + default: + return "" + } +} + +const ( + ConditionalBoundary MutantType = iota +) + +var tokenMutantType = map[token.Token][]MutantType{ + token.GTR: {ConditionalBoundary}, + token.LSS: {ConditionalBoundary}, + token.GEQ: {ConditionalBoundary}, + token.LEQ: {ConditionalBoundary}, +} + +var mutations = map[MutantType]map[token.Token]token.Token{ + ConditionalBoundary: { + token.GTR: token.GEQ, + token.LSS: token.LEQ, + token.GEQ: token.GTR, + token.LEQ: token.LSS, + }, +} diff --git a/mutator/mutator.go b/mutator/mutator.go new file mode 100644 index 00000000..d0829fc7 --- /dev/null +++ b/mutator/mutator.go @@ -0,0 +1,131 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mutator + +import ( + "github.com/k3rn31/gremlins/coverage" + "go/ast" + "go/parser" + "go/token" + "io" + "io/fs" + "path/filepath" + "strings" +) + +type MutationStatus int + +const ( + NotCovered MutationStatus = iota + Runnable + Lived + Killed +) + +func (ms MutationStatus) String() string { + switch ms { + case NotCovered: + return "NOT COVERED" + case Runnable: + return "RUNNABLE" + case Lived: + return "LIVED" + case Killed: + return "KILLED" + default: + panic("this should not happen") + } +} + +type Mutant struct { + Position token.Position + MutantType MutantType + Status MutationStatus + Token token.Token + Mutation token.Token +} + +type Mutator struct { + covProfile coverage.Profile + fs fs.FS +} + +func New(fs fs.FS, p coverage.Profile) *Mutator { + return &Mutator{covProfile: p, fs: fs} +} + +func (mu Mutator) Run() []Mutant { + var result []Mutant + _ = fs.WalkDir(mu.fs, ".", func(path string, d fs.DirEntry, err error) error { + if filepath.Ext(path) == ".go" && !strings.HasSuffix(path, "_test.go") { + src, _ := mu.fs.Open(path) + r := mu.runOnFile(path, src) + result = append(result, r...) + } + return nil + }) + + return result +} + +func (mu Mutator) runOnFile(fileName string, src io.Reader) []Mutant { + var result []Mutant + set := token.NewFileSet() + file, _ := parser.ParseFile(set, fileName, src, parser.ParseComments) + ast.Inspect(file, func(node ast.Node) bool { + switch node := node.(type) { + case *ast.BinaryExpr: + r, ok := mu.inspectBinaryExpr(set, node) + if !ok { + return true + } + result = append(result, r...) + } + return true + }) + return result +} + +func (mu Mutator) inspectBinaryExpr(set *token.FileSet, be *ast.BinaryExpr) ([]Mutant, bool) { + var result []Mutant + mutantTypes, ok := tokenMutantType[be.Op] + if !ok { + return nil, false + } + for _, mt := range mutantTypes { + pos := set.Position(be.OpPos) + mutant := Mutant{ + MutantType: mt, + Token: be.Op, + Mutation: mutations[mt][be.Op], + Status: mutationStatus(mu, pos), + Position: pos, + } + result = append(result, mutant) + } + + return result, true +} + +func mutationStatus(mu Mutator, pos token.Position) MutationStatus { + var status MutationStatus + if mu.covProfile.IsCovered(pos) { + status = Runnable + } + + return status +} diff --git a/mutator/mutator_test.go b/mutator/mutator_test.go new file mode 100644 index 00000000..fc9d5382 --- /dev/null +++ b/mutator/mutator_test.go @@ -0,0 +1,157 @@ +/* + * Copyright 2022 The Gremlins Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mutator_test + +import ( + "github.com/k3rn31/gremlins/coverage" + "github.com/k3rn31/gremlins/mutator" + "go/token" + "io/ioutil" + "os" + "strings" + "testing" + "testing/fstest" +) + +func coveredPosition(fixture string) coverage.Profile { + fn := filenameFromFixture(fixture) + return coverage.Profile{fn: {{StartLine: 6, EndLine: 7, StartCol: 8, EndCol: 9}}} +} + +func notCoveredPosition(fixture string) coverage.Profile { + fn := filenameFromFixture(fixture) + return coverage.Profile{fn: {{StartLine: 9, EndLine: 9, StartCol: 8, EndCol: 9}}} +} + +func TestMutations(t *testing.T) { + testCases := []struct { + name string + fixture string + mutantType mutator.MutantType + token token.Token + tokenMutation token.Token + covProfile coverage.Profile + mutStatus mutator.MutationStatus + }{ + { + name: "it recognizes CONDITIONAL_BOUNDARY with GTR", + fixture: "testdata/fixtures/conditional_boundary_gtr_go", + mutantType: mutator.ConditionalBoundary, + token: token.GTR, + tokenMutation: token.GEQ, + covProfile: coveredPosition("testdata/fixtures/conditional_boundary_gtr_go"), + mutStatus: mutator.Runnable, + }, + { + name: "it recognizes CONDITIONAL_BOUNDARY with LSS", + fixture: "testdata/fixtures/conditional_boundary_lss_go", + mutantType: mutator.ConditionalBoundary, + token: token.LSS, + tokenMutation: token.LEQ, + covProfile: notCoveredPosition("testdata/fixtures/conditional_boundary_lss_go"), + mutStatus: mutator.NotCovered, + }, + { + name: "it recognizes CONDITIONAL_BOUNDARY with LEQ", + fixture: "testdata/fixtures/conditional_boundary_leq_go", + mutantType: mutator.ConditionalBoundary, + token: token.LEQ, + tokenMutation: token.LSS, + covProfile: notCoveredPosition("testdata/fixtures/conditional_boundary_leq_go"), + mutStatus: mutator.NotCovered, + }, + { + name: "it recognizes CONDITIONAL_BOUNDARY with GEQ", + fixture: "testdata/fixtures/conditional_boundary_geq_go", + mutantType: mutator.ConditionalBoundary, + token: token.GEQ, + tokenMutation: token.GTR, + covProfile: notCoveredPosition("testdata/fixtures/conditional_boundary_geq_go"), + mutStatus: mutator.NotCovered, + }, + { + name: "it recognizes CONDITIONAL_BOUNDARY with recursion", + fixture: "testdata/fixtures/conditional_boundary_geq_land_true_go", + mutantType: mutator.ConditionalBoundary, + token: token.GEQ, + tokenMutation: token.GTR, + covProfile: notCoveredPosition("testdata/fixtures/conditional_boundary_geq_go"), + mutStatus: mutator.NotCovered, + }, + { + name: "it skips illegal CONDITIONAL_BOUNDARY", + fixture: "testdata/fixtures/conditional_boundary_illegal_go", + mutantType: mutator.ConditionalBoundary, + token: token.ILLEGAL, + covProfile: notCoveredPosition("testdata/fixtures/conditional_boundary_illegal_go"), + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f, _ := os.Open(tc.fixture) + src, _ := ioutil.ReadAll(f) + filename := filenameFromFixture(tc.fixture) + mapFS := fstest.MapFS{ + filename: {Data: src}, + } + mut := mutator.New(mapFS, tc.covProfile) + got := mut.Run() + + if tc.token == token.ILLEGAL { + if len(got) != 0 { + t.Errorf("expected no mutator found") + } + return + } + + for _, g := range got { + if g.MutantType == mutator.ConditionalBoundary && + g.Token == tc.token && + g.Mutation == tc.tokenMutation && + g.Status == tc.mutStatus { + return + } + } + + t.Errorf("expected mutations list to contain the found mutation") + t.Log(got) + }) + } +} + +func filenameFromFixture(fix string) string { + return strings.ReplaceAll(fix, "_go", ".go") +} + +func TestSkipTestAndNonGoFiles(t *testing.T) { + t.Parallel() + f, _ := os.Open("testdata/fixtures/conditional_boundary_geq_go") + file, _ := ioutil.ReadAll(f) + + sys := fstest.MapFS{ + "file_test.go": {Data: file}, + "folder1/file": {Data: file}, + } + mut := mutator.New(sys, nil) + got := mut.Run() + + if len(got) != 0 { + t.Errorf("should not receive results") + } +} diff --git a/mutator/testdata/fixtures/conditional_boundary_geq_go b/mutator/testdata/fixtures/conditional_boundary_geq_go new file mode 100644 index 00000000..25819fdc --- /dev/null +++ b/mutator/testdata/fixtures/conditional_boundary_geq_go @@ -0,0 +1,9 @@ +package main +import "fmt" +func main() { + a := 1 + b := 2 + if a >= 2 { + // Do something + } +} diff --git a/mutator/testdata/fixtures/conditional_boundary_geq_land_true_go b/mutator/testdata/fixtures/conditional_boundary_geq_land_true_go new file mode 100644 index 00000000..6b7811ce --- /dev/null +++ b/mutator/testdata/fixtures/conditional_boundary_geq_land_true_go @@ -0,0 +1,9 @@ +package main +import "fmt" +func main() { + a := 1 + b := 2 + if a >= 2 && true { + // Do something + } +} diff --git a/mutator/testdata/fixtures/conditional_boundary_gtr_go b/mutator/testdata/fixtures/conditional_boundary_gtr_go new file mode 100644 index 00000000..39379f16 --- /dev/null +++ b/mutator/testdata/fixtures/conditional_boundary_gtr_go @@ -0,0 +1,9 @@ +package main +import "fmt" +func main() { + a := 1 + b := 2 + if a > 2 { + // Do something + } +} diff --git a/mutator/testdata/fixtures/conditional_boundary_illegal_go b/mutator/testdata/fixtures/conditional_boundary_illegal_go new file mode 100644 index 00000000..09e8b32c --- /dev/null +++ b/mutator/testdata/fixtures/conditional_boundary_illegal_go @@ -0,0 +1,9 @@ +package main +import "fmt" +func main() { + a := 1 + b := 2 + if a << 2 { + // ILLEGAL + } +} diff --git a/mutator/testdata/fixtures/conditional_boundary_leq_go b/mutator/testdata/fixtures/conditional_boundary_leq_go new file mode 100644 index 00000000..774a0317 --- /dev/null +++ b/mutator/testdata/fixtures/conditional_boundary_leq_go @@ -0,0 +1,9 @@ +package main +import "fmt" +func main() { + a := 1 + b := 2 + if a <= 2 { + // Do something + } +} diff --git a/mutator/testdata/fixtures/conditional_boundary_lss_go b/mutator/testdata/fixtures/conditional_boundary_lss_go new file mode 100644 index 00000000..90a962a3 --- /dev/null +++ b/mutator/testdata/fixtures/conditional_boundary_lss_go @@ -0,0 +1,9 @@ +package main +import "fmt" +func main() { + a := 1 + b := 2 + if a < 2 { + // Do something + } +}