Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

e2e/util_test: enable tests to work using external endpoint #506

Merged
merged 30 commits into from
Sep 16, 2022
Merged

e2e/util_test: enable tests to work using external endpoint #506

Show file tree
Hide file tree
Changes from 29 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
dfdc8a3
refactor cleanup behavior
boraberke Aug 26, 2022
91c9fea
e2e: refactor cleanup behavior using t.Cleanup
boraberke Aug 26, 2022
e120ab1
Allow tests to use external endpoint url from env
boraberke Aug 26, 2022
c403925
add cleanup function to remove buckets after test
boraberke Aug 26, 2022
5ab6264
uniquify bucket names for all tests
boraberke Aug 31, 2022
8dfa8d9
Merge branch 'master' into change-test-endpoint-read-from-env
boraberke Aug 31, 2022
c5adc24
add unit test for addRandomSuffixTo func
boraberke Aug 31, 2022
8b129db
add unit test to addRandomSuffixTo func
boraberke Aug 31, 2022
32e867d
test github actions
boraberke Sep 1, 2022
ae372aa
test github actions with env
boraberke Sep 1, 2022
08f0f38
fix endpoint for env tests
boraberke Sep 1, 2022
726f5f4
change test structures to use unique bucket names
boraberke Sep 1, 2022
9837b50
refactor all tests to use unique bucket names
boraberke Sep 2, 2022
4301747
Merge branch 'peak:master' into change-test-endpoint-read-from-env
boraberke Sep 2, 2022
6708fcc
add test to s3BucketFromTestNameWithPrefix
boraberke Sep 2, 2022
480aaa1
refactor TestCatS3ObjectFail
boraberke Sep 2, 2022
208a81d
add slowDownRetryer
boraberke Sep 2, 2022
1775bee
Merge branch 'peak:master' into change-test-endpoint-read-from-env
boraberke Sep 2, 2022
ed1ea21
fix some of the failed tests when used with gcs
boraberke Sep 5, 2022
8e85e41
fix most of the failed tests on gcs
boraberke Sep 5, 2022
238547f
conclude path escape issues for with space char
boraberke Sep 6, 2022
7ebcf12
fix TestCopySingleFileToS3 not working in gcs
boraberke Sep 6, 2022
6c6eb98
fix TestAppProxy in gcs
boraberke Sep 6, 2022
ed663fc
vendor: update gofakes3 to v0.0.13
boraberke Sep 7, 2022
5005910
change chunk size to 1 when gcs is used
boraberke Sep 7, 2022
7527fa6
add S5CMD_IS_VIRTUAL_HOST
boraberke Sep 9, 2022
9e0fd43
fix reviews
boraberke Sep 9, 2022
9dc6e80
fix reviews
boraberke Sep 13, 2022
5450e1e
Merge branch 'peak:master' into change-test-endpoint-read-from-env
boraberke Sep 13, 2022
3bc7a91
Merge branch 'master' into change-test-endpoint-read-from-env
boraberke Sep 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions command/cp_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,22 @@ func TestGuessContentType(t *testing.T) {
`,
expectedContentType: "text/html; charset=utf-8",
},
// check file extension first without checking the content
{
filename: "index*.txt",
content: `
<!DOCTYPE html>
<html>
<head>
<title>Hello World</title>
</head>
<body>
<p>Hello, World! I am s5cmd :)</p>
</body>
</html>
`,
expectedContentType: "text/plain; charset=utf-8",
},
}

for _, tc := range testcases {
Expand Down
29 changes: 23 additions & 6 deletions e2e/app_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package e2e

import (
"fmt"
"net/url"
"os"
"strconv"
"strings"
Expand Down Expand Up @@ -76,7 +77,6 @@ func TestAppDashStat(t *testing.T) {
t.Parallel()

const (
bucket = "bucket"
fileContent = "this is a file content"
src = "file1.txt"
expectedOutputIfPrinted = "Operation\tTotal\tError\tSuccess\t"
Expand All @@ -87,19 +87,19 @@ func TestAppDashStat(t *testing.T) {
isPrintExpected bool
}{
{
command: fmt.Sprintf("--stat --log %v cp s3://bucket/%v .", "trace", src),
command: fmt.Sprintf("--stat --log %v ls", "trace"),
isPrintExpected: true,
},
{
command: fmt.Sprintf("--stat --log %v cp s3://bucket/%v .", "debug", src),
command: fmt.Sprintf("--stat --log %v ls", "debug"),
isPrintExpected: true,
},
{
command: fmt.Sprintf("--stat --log %v cp s3://bucket/%v .", "info", src),
command: fmt.Sprintf("--stat --log %v ls", "info"),
isPrintExpected: true,
},
{
command: fmt.Sprintf("--stat --log %v cp s3://bucket/%v .", "error", src),
command: fmt.Sprintf("--stat --log %v ls", "error"),
isPrintExpected: true,
},
// if level is an empty string, it ignores log levels and uses default.
Expand All @@ -118,8 +118,11 @@ func TestAppDashStat(t *testing.T) {
t.Parallel()
s3client, s5cmd := setup(t)

bucket := s3BucketFromTestName(t)

createBucket(t, s3client, bucket)
putFile(t, s3client, bucket, src, fileContent)

cmd := s5cmd(strings.Fields(tc.command)...)

result := icmd.RunCmd(cmd)
Expand All @@ -132,7 +135,6 @@ func TestAppDashStat(t *testing.T) {
}

func TestAppProxy(t *testing.T) {

testcases := []struct {
name string
flag string
Expand All @@ -154,6 +156,21 @@ func TestAppProxy(t *testing.T) {
proxy := httpProxy{}
pxyUrl := setupProxy(t, &proxy)

// set endpoint scheme to 'http'
if os.Getenv(s5cmdTestEndpointEnv) != "" {
origEndpoint := os.Getenv(s5cmdTestEndpointEnv)
endpoint, err := url.Parse(origEndpoint)
if err != nil {
t.Fatal(err)
}
endpoint.Scheme = "http"
os.Setenv(s5cmdTestEndpointEnv, endpoint.String())

defer func() {
os.Setenv(s5cmdTestEndpointEnv, origEndpoint)
}()
}

os.Setenv("http_proxy", pxyUrl)

_, s5cmd := setup(t, withProxy())
Expand Down
39 changes: 17 additions & 22 deletions e2e/cat_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,8 @@ func TestCatS3Object(t *testing.T) {
t.Parallel()

const (
bucket = "bucket"
filename = "file.txt"
)

src := fmt.Sprintf("s3://%v/%v", bucket, filename)
contents, expected := getSequentialFileContent()

testcases := []struct {
Expand All @@ -29,7 +26,6 @@ func TestCatS3Object(t *testing.T) {
name: "cat remote object",
cmd: []string{
"cat",
src,
},
expected: expected,
},
Expand All @@ -38,7 +34,6 @@ func TestCatS3Object(t *testing.T) {
cmd: []string{
"--json",
"cat",
src,
},
expected: expected,
assertOps: []assertOp{
Expand All @@ -53,9 +48,14 @@ func TestCatS3Object(t *testing.T) {

s3client, s5cmd := setup(t)

bucket := s3BucketFromTestName(t)

createBucket(t, s3client, bucket)
putFile(t, s3client, bucket, filename, contents)

src := fmt.Sprintf("s3://%v/%v", bucket, filename)
tc.cmd = append(tc.cmd, src)

cmd := s5cmd(tc.cmd...)
result := icmd.RunCmd(cmd)

Expand All @@ -70,67 +70,59 @@ func TestCatS3Object(t *testing.T) {
func TestCatS3ObjectFail(t *testing.T) {
t.Parallel()

const (
bucket = "bucket"
filename = "file.txt"
)

bucketSrc := fmt.Sprintf("s3://%v", bucket)
prefixSrc := fmt.Sprintf("%v/prefix", bucketSrc)
src := fmt.Sprintf("%s/%v", prefixSrc, filename)

testcases := []struct {
src string
name string
cmd []string
expected map[int]compareFunc
assertOps []assertOp
}{
{
src: "s3://%v/prefix/file.txt",
name: "cat non existent remote object",
cmd: []string{
"cat",
src,
},
expected: map[int]compareFunc{
0: contains(`ERROR "cat s3://bucket/prefix/file.txt": NoSuchKey: status code: 404`),
0: match(`ERROR "cat s3://(.*)/prefix/file\.txt": NoSuchKey:`),
},
},
{
src: "s3://%v/prefix/file.txt",
name: "cat non existent remote object with json flag",
cmd: []string{
"--json",
"cat",
src,
},
expected: map[int]compareFunc{
0: contains(`{"operation":"cat","command":"cat s3://bucket/prefix/file.txt","error":"NoSuchKey: status code: 404,`),
0: match(`{"operation":"cat","command":"cat s3:\/\/(.*)\/prefix\/file\.txt","error":"NoSuchKey:`),
},
assertOps: []assertOp{
jsonCheck(true),
},
},
{
src: "s3://%v/prefix/file.txt/*",
name: "cat remote object with glob",
cmd: []string{
"--json",
"cat",
src + "/*",
},
expected: map[int]compareFunc{
0: equals(`{"operation":"cat","command":"cat s3://bucket/prefix/file.txt/*","error":"remote source \"s3://bucket/prefix/file.txt/*\" can not contain glob characters"}`),
0: match(`{"operation":"cat","command":"cat s3:\/\/(.+)?\/prefix\/file\.txt\/\*","error":"remote source \\"s3:\/\/(.*)\/prefix\/file\.txt\/\*\\" can not contain glob characters"}`),
},
assertOps: []assertOp{
jsonCheck(true),
},
},
{
src: "s3://%v/prefix/",
name: "cat bucket",
cmd: []string{
"cat",
bucketSrc,
},
expected: map[int]compareFunc{
0: contains(`ERROR "cat s3://bucket": remote source must be an object`),
0: match(`ERROR "cat s3://(.+)?": remote source must be an object`),
},
},
}
Expand All @@ -141,9 +133,12 @@ func TestCatS3ObjectFail(t *testing.T) {
t.Parallel()
s3client, s5cmd := setup(t)

bucket := s3BucketFromTestName(t)
createBucket(t, s3client, bucket)

tc.cmd = append(tc.cmd, fmt.Sprintf(tc.src, bucket))
cmd := s5cmd(tc.cmd...)

result := icmd.RunCmd(cmd)

result.Assert(t, icmd.Expected{ExitCode: 1})
Expand Down
Loading