From f7bb5302d350a4e46383e7a12d315d57caea2f9b Mon Sep 17 00:00:00 2001 From: Richard Gomez Date: Sun, 15 Dec 2024 21:47:31 -0500 Subject: [PATCH] fix(azuredevops): detector wasn't working --- .../azure_devops/personalaccesstoken.go | 167 ++++++++++++++++++ .../personalaccesstoken_integration_test.go} | 2 +- .../azure_devops/personalaccesstoken_test.go | 128 ++++++++++++++ .../azuredevopspersonalaccesstoken.go | 101 ----------- .../azuredevopspersonalaccesstoken_test.go | 88 --------- pkg/engine/defaults/defaults.go | 4 +- 6 files changed, 298 insertions(+), 192 deletions(-) create mode 100644 pkg/detectors/azure_devops/personalaccesstoken.go rename pkg/detectors/{azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go => azure_devops/personalaccesstoken_integration_test.go} (99%) create mode 100644 pkg/detectors/azure_devops/personalaccesstoken_test.go delete mode 100644 pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go delete mode 100644 pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go diff --git a/pkg/detectors/azure_devops/personalaccesstoken.go b/pkg/detectors/azure_devops/personalaccesstoken.go new file mode 100644 index 0000000000000..0c937f0de4e33 --- /dev/null +++ b/pkg/detectors/azure_devops/personalaccesstoken.go @@ -0,0 +1,167 @@ +package azure_devops + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + regexp "github.com/wasilibs/go-re2" + + "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct { + client *http.Client + detectors.DefaultMultiPartCredentialProvider +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_AzureDevopsPersonalAccessToken +} + +func (s Scanner) Description() string { + return "Azure DevOps is a suite of development tools provided by Microsoft. Personal Access Tokens (PATs) are used to authenticate and authorize access to Azure DevOps services and resources." +} + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"dev.azure.com", "az devops"} +} + +var ( + // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure", "az", "token", "pat"}) + `\b([a-z0-9]{52}|[a-zA-Z0-9]{84})\b`) + orgPat = regexp.MustCompile(`dev\.azure\.com/([0-9a-zA-Z][0-9a-zA-Z-]{5,48}[0-9a-zA-Z])\b`) + + invalidOrgCache = simple.NewCache[struct{}]() +) + +// FromData will find and optionally verify AzureDevopsPersonalAccessToken secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + // Deduplicate results. + keyMatches := make(map[string]struct{}) + for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) { + m := match[1] + if detectors.StringShannonEntropy(m) < 3 { + continue + } + keyMatches[m] = struct{}{} + } + orgMatches := make(map[string]struct{}) + for _, match := range orgPat.FindAllStringSubmatch(dataStr, -1) { + m := match[1] + if invalidOrgCache.Exists(m) { + continue + } + orgMatches[m] = struct{}{} + } + + for key := range keyMatches { + for org := range orgMatches { + r := detectors.Result{ + DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, + Raw: []byte(key), + RawV2: []byte(fmt.Sprintf(`{"organization":"%s","token":"%s"}`, org, key)), + } + + if verify { + if s.client == nil { + s.client = common.SaneHttpClient() + } + + isVerified, extraData, verificationErr := verifyMatch(ctx, s.client, org, key) + r.Verified = isVerified + r.ExtraData = extraData + if verificationErr != nil { + if errors.Is(verificationErr, errInvalidOrg) { + delete(orgMatches, org) + invalidOrgCache.Set(org, struct{}{}) + continue + } + r.SetVerificationError(verificationErr) + } + } + + results = append(results, r) + } + } + + return results, nil +} + +var errInvalidOrg = errors.New("invalid organization") + +func verifyMatch(ctx context.Context, client *http.Client, org string, key string) (bool, map[string]string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "https://dev.azure.com/"+org+"/_apis/projects", nil) + if err != nil { + return false, nil, err + } + + req.SetBasicAuth("", key) + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + res, err := client.Do(req) + if err != nil { + return false, nil, err + } + defer func() { + _, _ = io.Copy(io.Discard, res.Body) + _ = res.Body.Close() + }() + + switch res.StatusCode { + case http.StatusOK: + // {"count":1,"value":[{"id":"...","name":"Test","url":"https://dev.azure.com/...","state":"wellFormed","revision":11,"visibility":"private","lastUpdateTime":"2024-12-16T02:23:58.86Z"}]} + var projectsRes listProjectsResponse + if json.NewDecoder(res.Body).Decode(&projectsRes) != nil { + return false, nil, err + } + + // Condense a list of organizations + roles. + var ( + extraData map[string]string + projects = make([]string, 0, len(projectsRes.Value)) + ) + for _, p := range projectsRes.Value { + projects = append(projects, p.Name) + } + if len(projects) > 0 { + extraData = map[string]string{ + "projects": strings.Join(projects, ","), + } + } + return true, extraData, nil + case http.StatusUnauthorized: + // The secret is determinately not verified (nothing to do) + return false, nil, nil + case http.StatusNotFound: + // Org doesn't exist. + return false, nil, errInvalidOrg + default: + body, _ := io.ReadAll(res.Body) + return false, nil, fmt.Errorf("unexpected HTTP response: status=%d, body=%q", res.StatusCode, string(body)) + } +} + +type listProjectsResponse struct { + Count int `json:"count"` + Value []projectResponse `json:"value"` +} + +type projectResponse struct { + Id string `json:"id"` + Name string `json:"name"` +} diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go b/pkg/detectors/azure_devops/personalaccesstoken_integration_test.go similarity index 99% rename from pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go rename to pkg/detectors/azure_devops/personalaccesstoken_integration_test.go index 2b7c68f3fd677..793e6e685b3dd 100644 --- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go +++ b/pkg/detectors/azure_devops/personalaccesstoken_integration_test.go @@ -1,7 +1,7 @@ //go:build detectors // +build detectors -package azuredevopspersonalaccesstoken +package azure_devops import ( "context" diff --git a/pkg/detectors/azure_devops/personalaccesstoken_test.go b/pkg/detectors/azure_devops/personalaccesstoken_test.go new file mode 100644 index 0000000000000..c3d94813da794 --- /dev/null +++ b/pkg/detectors/azure_devops/personalaccesstoken_test.go @@ -0,0 +1,128 @@ +package azure_devops + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" +) + +func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) { + d := Scanner{} + ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) + + tests := []struct { + name string + input string + want []string + }{ + // old + { + name: "valid - old token", + input: ` +provider "azuredevops" { + # Configuration options + org_service_url = "https://dev.azure.com/housemd" + personal_access_token = "qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a" +}`, + want: []string{"qkfon5cdjdekin4qnkgfr2nf367h6yjnnqm5upwqepd3rekl4l5a:housemd"}, + }, + + // new + { + name: "valid - az devops CLI", + input: ` echo "Tests failed. Creating a bug in Azure DevOps..." + az devops login --organization https://dev.azure.com/TechServicesCorp --token A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB + az boards work-item create --title "Automated Bug: Test Failure" --type $(bugType) --description "Tests failed. See results.log for details." --project "Test"`, + want: []string{"A0us9bS1c6qe5blb6CT4FGRR4JcmPDg7uadVFmw4D65bvtdPcdVdJQQJ99AKACAAAAAPnX9AAAASAhDO4GFB:TechServicesCorp"}, + }, + { + name: "valid - environment variables", + input: `# Base image: Azure CLI with a lightweight Ubuntu distribution-mcr.microsoft.com/azure-cli:2.52.0 +FROM ubuntu:20.04 + +# Set environment variables for Azure DevOps agent +ENV AZP_URL=https://dev.azure.com/EBOrg21 +ENV AZP_TOKEN=2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2 +ENV AZP_POOL=TestParty +`, + want: []string{"2ZGS1XLyxTU2wXlrXy71ldl1tBKceXM9kl6mVAeQchvWIErzkwtBJQjJ99AKACAAAAAAAAAAAAASAZDO5BA2:EBOrg21"}, + }, + { + name: "valid - jupyter notebook", + input: ` "4 https://dev.azure.com/SSGL-SMT/10_BG_AU5... " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "token = r\"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm\"" + ]`, + want: []string{"49QzGd2ZOLTWdoMc0S3M0cZkVVsBMTua01tlMYOkTUnEwxebgYdheQQJ99AKACAAAAAHsyrdAAASAZDOULjm:SSGL-SMT"}, + }, + + // Invalid + { + name: "invalid", + input: `ssh.dev.azure.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H`, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) + if len(matchedDetectors) == 0 { + t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) + return + } + + results, err := d.FromData(context.Background(), false, []byte(test.input)) + if err != nil { + t.Errorf("error = %v", err) + return + } + + if len(results) != len(test.want) { + if len(results) == 0 { + t.Errorf("did not receive result") + } else { + t.Errorf("expected %d results, only received %d", len(test.want), len(results)) + } + return + } + + actual := make(map[string]struct{}, len(results)) + for _, r := range results { + if len(r.RawV2) > 0 { + actual[string(r.RawV2)] = struct{}{} + } else { + actual[string(r.Raw)] = struct{}{} + } + } + expected := make(map[string]struct{}, len(test.want)) + for _, v := range test.want { + expected[v] = struct{}{} + } + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) + } + }) + } +} diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go deleted file mode 100644 index 3489d48606e80..0000000000000 --- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken.go +++ /dev/null @@ -1,101 +0,0 @@ -package azuredevopspersonalaccesstoken - -import ( - "context" - "fmt" - "net/http" - "strings" - - regexp "github.com/wasilibs/go-re2" - - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" -) - -type Scanner struct { - client *http.Client - detectors.DefaultMultiPartCredentialProvider -} - -// Ensure the Scanner satisfies the interface at compile time. -var _ detectors.Detector = (*Scanner)(nil) - -var ( - defaultClient = common.SaneHttpClient() - // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-z]{52})\b`) - orgPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z][0-9a-zA-Z-]{5,48}[0-9a-zA-Z])\b`) -) - -// Keywords are used for efficiently pre-filtering chunks. -// Use identifiers in the secret preferably, or the provider name. -func (s Scanner) Keywords() []string { - return []string{"azure"} -} - -// FromData will find and optionally verify AzureDevopsPersonalAccessToken secrets in a given set of bytes. -func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { - dataStr := string(data) - - matches := keyPat.FindAllStringSubmatch(dataStr, -1) - orgMatches := orgPat.FindAllStringSubmatch(dataStr, -1) - - for _, match := range matches { - if len(match) != 2 { - continue - } - resMatch := strings.TrimSpace(match[1]) - for _, orgMatch := range orgMatches { - if len(orgMatch) != 2 { - continue - } - resOrgMatch := strings.TrimSpace(orgMatch[1]) - - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken, - Raw: []byte(resMatch), - RawV2: []byte(resMatch + resOrgMatch), - } - - if verify { - client := s.client - if client == nil { - client = defaultClient - } - req, err := http.NewRequestWithContext(ctx, "GET", "https://dev.azure.com/"+resOrgMatch+"/_apis/projects", nil) - if err != nil { - continue - } - req.SetBasicAuth("", resMatch) - res, err := client.Do(req) - if err == nil { - defer res.Body.Close() - hasVerifiedRes, _ := common.ResponseContainsSubstring(res.Body, "lastUpdateTime") - if res.StatusCode >= 200 && res.StatusCode < 300 && hasVerifiedRes { - s1.Verified = true - } else if res.StatusCode == 401 { - // The secret is determinately not verified (nothing to do) - } else { - err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) - s1.SetVerificationError(err, resMatch) - } - } else { - s1.SetVerificationError(err, resMatch) - } - } - - results = append(results, s1) - } - } - - return results, nil -} - -func (s Scanner) Type() detectorspb.DetectorType { - return detectorspb.DetectorType_AzureDevopsPersonalAccessToken -} - -func (s Scanner) Description() string { - return "Azure DevOps is a suite of development tools provided by Microsoft. Personal Access Tokens (PATs) are used to authenticate and authorize access to Azure DevOps services and resources." -} diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go deleted file mode 100644 index b19ef9de3f554..0000000000000 --- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package azuredevopspersonalaccesstoken - -import ( - "context" - "testing" - - "github.com/google/go-cmp/cmp" - - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" - "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick" -) - -var ( - validPattern = ` - azure: - azure_key: uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un - azure_org_id: WOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG - ` - invalidPattern = ` - azure: - azure_key: uie5tff7m5H5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un - azure_org_id: LOKi - ` -) - -func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) { - d := Scanner{} - ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d}) - - tests := []struct { - name string - input string - want []string - }{ - { - name: "valid pattern", - input: validPattern, - want: []string{"uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8unWOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG"}, - }, - { - name: "invalid pattern", - input: invalidPattern, - want: nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input)) - if len(matchedDetectors) == 0 { - t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input) - return - } - - results, err := d.FromData(context.Background(), false, []byte(test.input)) - if err != nil { - t.Errorf("error = %v", err) - return - } - - if len(results) != len(test.want) { - if len(results) == 0 { - t.Errorf("did not receive result") - } else { - t.Errorf("expected %d results, only received %d", len(test.want), len(results)) - } - return - } - - actual := make(map[string]struct{}, len(results)) - for _, r := range results { - if len(r.RawV2) > 0 { - actual[string(r.RawV2)] = struct{}{} - } else { - actual[string(r.Raw)] = struct{}{} - } - } - expected := make(map[string]struct{}, len(test.want)) - for _, v := range test.want { - expected[v] = struct{}{} - } - - if diff := cmp.Diff(expected, actual); diff != "" { - t.Errorf("%s diff: (-want +got)\n%s", test.name, diff) - } - }) - } -} diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go index 75ce9fdd9b389..22362df912a00 100644 --- a/pkg/engine/defaults/defaults.go +++ b/pkg/engine/defaults/defaults.go @@ -64,12 +64,12 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aylien" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ayrshare" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_batch" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_devops" azure_serviceprincipal_v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal/v1" azure_serviceprincipal_v2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal/v2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_openai" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_storage" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurecontainerregistry" - "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuredevopspersonalaccesstoken" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchadminkey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchquerykey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bannerbear" @@ -894,7 +894,7 @@ func buildDetectorList() []detectors.Detector { &azure_serviceprincipal_v2.Scanner{}, &azure_batch.Scanner{}, &azurecontainerregistry.Scanner{}, - &azuredevopspersonalaccesstoken.Scanner{}, + &azure_devops.Scanner{}, // &azurefunctionkey.Scanner{}, // detector is throwing some FPs &azure_openai.Scanner{}, &azuresearchadminkey.Scanner{},