diff --git a/pkg/common/http.go b/pkg/common/http.go index 782ac1747c7b..f040f682232a 100644 --- a/pkg/common/http.go +++ b/pkg/common/http.go @@ -89,7 +89,7 @@ type CustomTransport struct { T http.RoundTripper } -func userAgent() string { +func UserAgent() string { if len(feature.UserAgentSuffix.Load()) > 0 { return "TruffleHog " + feature.UserAgentSuffix.Load() } @@ -97,7 +97,7 @@ func userAgent() string { } func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) { - req.Header.Add("User-Agent", userAgent()) + req.Header.Add("User-Agent", UserAgent()) return t.T.RoundTrip(req) } diff --git a/pkg/detectors/aws/access_keys/accesskey.go b/pkg/detectors/aws/access_keys/accesskey.go index cd92fc0ed94e..1e2e830c2278 100644 --- a/pkg/detectors/aws/access_keys/accesskey.go +++ b/pkg/detectors/aws/access_keys/accesskey.go @@ -3,9 +3,13 @@ package access_keys import ( "context" "fmt" + "net" "net/http" "strings" + "time" + "github.com/aws/aws-sdk-go-v2/aws/middleware" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/sts" @@ -19,7 +23,7 @@ import ( ) type scanner struct { - verificationClient *http.Client + verificationClient config.HTTPClient skipIDs map[string]struct{} detectors.DefaultMultiPartCredentialProvider } @@ -54,7 +58,6 @@ var _ interface { } = (*scanner)(nil) var ( - defaultVerificationClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. // Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids @@ -71,9 +74,28 @@ func (s scanner) Keywords() []string { } } -func (s scanner) getClient() *http.Client { +// The recommended way by AWS is to use the SDK's http client. +// https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-http.html +// Note: Using default http.Client causes SignatureInvalid error in response. therefore, based on http default client implementation, we are using the same configuration. +func getDefaultBuildableClient() *awshttp.BuildableClient { + return awshttp.NewBuildableClient(). + WithTimeout(common.DefaultResponseTimeout). + WithDialerOptions(func(dialer *net.Dialer) { + dialer.Timeout = 2 * time.Second + dialer.KeepAlive = 5 * time.Second + }). + WithTransportOptions(func(tr *http.Transport) { + tr.Proxy = http.ProxyFromEnvironment + tr.MaxIdleConns = 5 + tr.IdleConnTimeout = 5 * time.Second + tr.TLSHandshakeTimeout = 3 * time.Second + tr.ExpectContinueTimeout = 1 * time.Second + }) +} + +func (s scanner) getAWSBuilableClient() config.HTTPClient { if s.verificationClient == nil { - s.verificationClient = defaultVerificationClient + s.verificationClient = getDefaultBuildableClient() } return s.verificationClient } @@ -208,7 +230,7 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str // Prep AWS Creds for STS cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region), - config.WithHTTPClient(s.getClient()), + config.WithHTTPClient(s.getAWSBuilableClient()), config.WithCredentialsProvider( credentials.NewStaticCredentialsProvider(resIDMatch, resSecretMatch, ""), ), @@ -217,7 +239,9 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str return false, nil, err } // Create STS client - stsClient := sts.NewFromConfig(cfg) + stsClient := sts.NewFromConfig(cfg, func(o *sts.Options) { + o.APIOptions = append(o.APIOptions, middleware.AddUserAgentKeyValue("User-Agent", common.UserAgent())) + }) // Make the GetCallerIdentity API call resp, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) diff --git a/pkg/detectors/aws/access_keys/accesskey_integration_test.go b/pkg/detectors/aws/access_keys/accesskey_integration_test.go index a2acaf5a2f18..ef01ddad9a6e 100644 --- a/pkg/detectors/aws/access_keys/accesskey_integration_test.go +++ b/pkg/detectors/aws/access_keys/accesskey_integration_test.go @@ -5,11 +5,12 @@ package access_keys import ( "context" - "crypto/sha256" "fmt" + "sort" "testing" "time" + "github.com/brianvoe/gofakeit/v7" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" @@ -95,9 +96,8 @@ func TestAWS_FromChunk(t *testing.T) { id := testSecrets.MustGetField("AWS_ID") inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE") inactiveID := id[:len(id)-3] + "XYZ" - hasher := sha256.New() - hasher.Write([]byte(inactiveSecret)) - hash := string(hasher.Sum(nil)) + + hash := gofakeit.Password(true, true, true, false, false, 10) type args struct { ctx context.Context @@ -233,15 +233,6 @@ func TestAWS_FromChunk(t *testing.T) { "user_id": "AIDAZAVB57H5V3Q4ACRGM", }, }, - { - DetectorType: detectorspb.DetectorType_AWS, - Verified: false, - Redacted: inactiveID, - ExtraData: map[string]string{ - "account": "619888638459", - "resource_type": "Access key", - }, - }, }, wantErr: false, }, @@ -387,6 +378,15 @@ func TestAWS_FromChunk(t *testing.T) { verify: true, }, want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_AWS, + Verified: false, + Redacted: "AKIAZAVB57H55F3T4BKH", + ExtraData: map[string]string{ + "resource_type": "Access key", + "account": "619888638459", + }, + }, { DetectorType: detectorspb.DetectorType_AWS, Verified: true, @@ -425,6 +425,8 @@ func TestAWS_FromChunk(t *testing.T) { return x.Redacted < y.Redacted }), } + + sortResults(tt.want) if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" { t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } @@ -432,6 +434,13 @@ func TestAWS_FromChunk(t *testing.T) { } } +// Helper function to sort results due to the order of the redacted +func sortResults(results []detectors.Result) { + sort.SliceStable(results, func(i, j int) bool { + return results[i].Redacted < results[j].Redacted + }) +} + func BenchmarkFromData(benchmark *testing.B) { ctx := context.Background() s := scanner{} diff --git a/pkg/detectors/aws/access_keys/canary.go b/pkg/detectors/aws/access_keys/canary.go index 652340fa36d2..d94225a4eced 100644 --- a/pkg/detectors/aws/access_keys/canary.go +++ b/pkg/detectors/aws/access_keys/canary.go @@ -5,9 +5,11 @@ import ( "strings" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/sns" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" ) const thinkstMessage = "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries" @@ -49,7 +51,7 @@ func (s scanner) verifyCanary(ctx context.Context, resIDMatch, resSecretMatch st // Prep AWS Creds for SNS cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region), - config.WithHTTPClient(s.getClient()), + config.WithHTTPClient(s.getAWSBuilableClient()), config.WithCredentialsProvider( credentials.NewStaticCredentialsProvider(resIDMatch, resSecretMatch, ""), ), @@ -57,7 +59,9 @@ func (s scanner) verifyCanary(ctx context.Context, resIDMatch, resSecretMatch st if err != nil { return false, "", err } - svc := sns.NewFromConfig(cfg) + svc := sns.NewFromConfig(cfg, func(o *sns.Options) { + o.APIOptions = append(o.APIOptions, middleware.AddUserAgentKeyValue("User-Agent", common.UserAgent())) + }) // Prep vars and Publish to SNS _, err = svc.Publish(ctx, &sns.PublishInput{