Skip to content

Commit

Permalink
Add testing each version
Browse files Browse the repository at this point in the history
  • Loading branch information
@tenntenn
tenntenn committed Sep 10, 2020
1 parent 324717c commit 7d1bced
Show file tree
Hide file tree
Showing 3 changed files with 229 additions and 43 deletions.
5 changes: 4 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ module github.com/gostaticanalysis/testutil
go 1.15

require (
github.com/hashicorp/go-version v1.2.1
github.com/otiai10/copy v1.2.0
github.com/rogpeppe/go-internal v1.6.2
github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3
golang.org/x/mod v0.3.0
golang.org/x/text v0.3.2
golang.org/x/tools v0.0.0-20200908211811-12e1bf57a112
)
18 changes: 11 additions & 7 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=
github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc=
github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
github.com/rogpeppe/go-internal v1.6.2 h1:aIihoIOHCiLZHxyoNQ+ABL4NKhFTgKLBdMLyEAh98m0=
github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/tenntenn/text v1.0.0 h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=
github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag=
github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand All @@ -23,13 +25,15 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200908211811-12e1bf57a112 h1:DmrRJy1qn9VDMf4+GSpRlwfZ51muIF7r96MFBFP4bPM=
golang.org/x/tools v0.0.0-20200908211811-12e1bf57a112/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
249 changes: 214 additions & 35 deletions mod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,43 +2,41 @@ package testutil

import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"sync"
"testing"

"github.com/hashicorp/go-version"
"github.com/otiai10/copy"
"github.com/rogpeppe/go-internal/modfile"
tnntransform "github.com/tenntenn/text/transform"
"golang.org/x/mod/modfile"
"golang.org/x/text/transform"
"golang.org/x/tools/go/analysis"
"golang.org/x/tools/go/analysis/analysistest"
)

// WithModules creates a temp dir which is copied from baseDir and generates vendor directory with go.mod.
// WithModules creates a temp dir which is copied from srcdir and generates vendor directory with go.mod.
// go.mod can be specified by modfileReader.
func WithModules(t *testing.T, baseDir string, modfileReader io.Reader) (dir string) {
// Example:
// func TestAnalyzer(t *testing.T) {
// testdata := testutil.WithModules(t, analysistest.TestData(), nil)
// analysistest.Run(t, testdata, sample.Analyzer, "a")
// }
func WithModules(t *testing.T, srcdir string, modfile io.Reader) (dir string) {
t.Helper()
dir = t.TempDir()
if err := copy.Copy(baseDir, dir); err != nil {
if err := copy.Copy(srcdir, dir); err != nil {
t.Fatal("cannot copy a directory:", err)
}

if modfileReader != nil {
fn := filepath.Join(dir, "go.mod")
f, err := os.Create(fn)
if err != nil {
t.Fatal("cannot create go.mod:", err)
}

if _, err := io.Copy(f, modfileReader); err != nil {
t.Fatal("cannot create go.mod:", err)
}

if err := f.Close(); err != nil {
t.Fatal("cannot close go.mod", err)
}
}

var ok bool
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
Expand All @@ -54,19 +52,26 @@ func WithModules(t *testing.T, baseDir string, modfileReader io.Reader) (dir str
}

for _, file := range files {
if file.Name() != "go.mod" {
continue
}
if file.Name() == "go.mod" {
if modfile != nil {
fn := filepath.Join(path, "go.mod")
f, err := os.Create(fn)
if err != nil {
t.Fatal("cannot create go.mod:", err)
}

cmd := exec.Command("go", "mod", "vendor")
cmd.Stdout = ioutil.Discard
var errBuf bytes.Buffer
cmd.Stderr = &errBuf
cmd.Dir = path
if err := cmd.Run(); err != nil {
return fmt.Errorf("%w: %s", err, &errBuf)
if _, err := io.Copy(f, modfile); err != nil {
t.Fatal("cannot create go.mod:", err)
}

if err := f.Close(); err != nil {
t.Fatal("cannot close go.mod", err)
}
}
execCmd(t, path, "go", "mod", "vendor")
ok = true
return nil
}
break
}

return nil
Expand All @@ -75,18 +80,32 @@ func WithModules(t *testing.T, baseDir string, modfileReader io.Reader) (dir str
t.Fatal("go mod vendor:", err)
}

if !ok {
t.Fatal("does not find go.mod")
}

return dir
}

// ModFile opens a mod file and fixes versions by the version fixer.
func ModFile(t *testing.T, modfilePath string, fix modfile.VersionFixer) io.Reader {
// ModFile opens a mod file with the path and fixes versions by the version fixer.
// If the path is direcotry, ModFile opens go.mod which is under the path.
func ModFile(t *testing.T, path string, fix modfile.VersionFixer) io.Reader {
t.Helper()
data, err := ioutil.ReadFile(modfilePath)

info, err := os.Stat(path)
if err != nil {
t.Fatal("cannot get stat of path:", err)
}
if info.IsDir() {
path = filepath.Join(path, "go.mod")
}

data, err := ioutil.ReadFile(path)
if err != nil {
t.Fatal("cannot read go.mod:", err)
}

f, err := modfile.Parse(modfilePath, data, fix)
f, err := modfile.Parse(path, data, fix)
if err != nil {
t.Fatal("cannot parse go.mod:", err)
}
Expand All @@ -98,3 +117,163 @@ func ModFile(t *testing.T, modfilePath string, fix modfile.VersionFixer) io.Read

return bytes.NewReader(out)
}

// ModuleVersion has module path and its version.
type ModuleVersion struct {
Module string
Version string
}

// String implements fmt.Stringer.
func (modver ModuleVersion) String() string {
return fmt.Sprintf("%s@%s", modver.Module, modver.Version)
}

// AllVersion get available all versions of the module.
func AllVersion(t *testing.T, module string) []ModuleVersion {
t.Helper()

dir := t.TempDir()
execCmd(t, dir, "go", "mod", "init", "tmp")
r := execCmd(t, dir, "go", "list", "-m", "-versions", "-json", module)
var v struct{ Versions []string }
if err := json.NewDecoder(r).Decode(&v); err != nil {
t.Fatal("cannot decode JSON", err)
}

vers := make([]ModuleVersion, len(v.Versions))
for i := range v.Versions {
vers[i] = ModuleVersion{
Module: module,
Version: v.Versions[i],
}
}

return vers
}

// FilterVersion returns versions of the module which satisfy the constraints such as ">= v2.0.0"
// The constraints rule uses github.com/hashicorp/go-version.
//
// Example:
// func TestAnalyzer(t *testing.T) {
// vers := FilterVersion(t, "github.com/tenntenn/greeting/v2", ">= v2.0.0")
// RunWithVersions(t, analysistest.TestData(), mod.Analyzer, vers, "a")
// }
func FilterVersion(t *testing.T, module, constraints string) []ModuleVersion {
t.Helper()

c, err := version.NewConstraint(constraints)
if err != nil {
t.Fatal("cannot parse constraints", err)
}

var vers []ModuleVersion
for _, ver := range AllVersion(t, module) {
v, err := version.NewVersion(ver.Version)
if err != nil {
t.Fatal("cannot parse version", err)
}
if c.Check(v) {
vers = append(vers, ver)
}
}

return vers
}

// RunWithVersions runs analysistest.Run with modules which version is specified the vers.
//
// Example:
// func TestAnalyzer(t *testing.T) {
// vers := AllVersion(t, "github.com/tenntenn/greeting/v2")
// RunWithVersions(t, analysistest.TestData(), mod.Analyzer, vers, "a")
// }
//
// The test run in temporary directory which is isolated the dir.
// analysistest.Run uses packages.Load and it prints errors into os.Stderr.
// Becase the error messages include the temporary directory path, so RunWithVersions replaces os.Stderr.
// Replacing os.Stderr is not thread safe.
// If you want to turn off replacing os.Stderr, you can use ReplaceStderr(false).
func RunWithVersions(t *testing.T, dir string, a *analysis.Analyzer, vers []ModuleVersion, pkg string) map[ModuleVersion][]*analysistest.Result {
path := filepath.Join(dir, "src", pkg)

results := make(map[ModuleVersion][]*analysistest.Result, len(vers))
for _, modver := range vers {
modver := modver
t.Run(modver.String(), func(t *testing.T) {
modfile := ModFile(t, path, func(module, ver string) (string, error) {
if modver.Module == module {
return modver.Version, nil
}
return ver, nil
})
tmpdir := WithModules(t, dir, modfile)
replaceStderr(t, tmpdir, dir)
results[modver] = analysistest.Run(t, tmpdir, a, pkg)
})
}

return results
}

func execCmd(t *testing.T, dir, cmd string, args ...string) io.Reader {
var stdout, stderr bytes.Buffer
_cmd := exec.Command(cmd, args...)
_cmd.Stdout = &stdout
_cmd.Stderr = &stderr
_cmd.Dir = dir
if err := _cmd.Run(); err != nil {
t.Fatal(err, "\n", &stderr)
}
return &stdout
}

var (
stderrMutex sync.RWMutex
doNotUseFilteredStderr bool
)

func ReplaceStderr(onoff bool) {
stderrMutex.Lock()
doNotUseFilteredStderr = !onoff
stderrMutex.Unlock()
}

func replaceStderr(t *testing.T, old, new string) {
stderrMutex.RLock()
ok := !doNotUseFilteredStderr
stderrMutex.RUnlock()
if !ok {
return
}

r, w, err := os.Pipe()
if err != nil {
t.Fatal("cannot create pipe", err)
}

origStderr := os.Stderr
stderrMutex.Lock()
os.Stderr = w
stderrMutex.Unlock()
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(func() {
cancel()
stderrMutex.Lock()
os.Stderr = origStderr
stderrMutex.Unlock()
})

go func() {
t := tnntransform.ReplaceString(old, new)
w := transform.NewWriter(origStderr, t)
for {
select {
case <-ctx.Done():
default:
io.CopyN(w, r, 1024)
}
}
}()
}

0 comments on commit 7d1bced

Please sign in to comment.