Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Azure: reduce false-positive secrets #3722

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Azure: reduce false-positive secrets #3722

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 34 additions & 15 deletions pkg/detectors/azure_entra/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"io"
"net/http"
stdRegexp "regexp" // Faster for small inputs.
"strings"

regexp "github.com/wasilibs/go-re2"
Expand All @@ -24,7 +25,7 @@ var (
// https://learn.microsoft.com/en-us/microsoft-365/admin/setup/domains-faq?view=o365-worldwide#why-do-i-have-an--onmicrosoft-com--domain
tenantIdPat = regexp.MustCompile(fmt.Sprintf(
//language=regexp
`(?i)(?:(?:login\.microsoftonline\.com/|(?:login|sts)\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,60}?)(%s)|https?://(%s)|X-AnchorMailbox(?:.|\s){0,60}?@(%s)|/(%s)/(?:oauth2/v2\.0|B2C_1\w+|common|discovery|federationmetadata|kerberos|login|openid/|reprocess|resume|saml2|token|uxlogout|v2\.0|wsfed))`,
`(?i)(?:(?:login\.microsoft(?:online)?\.com/|(?:login|sts)\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,60}?)(%s)|https?://(%s)|X-AnchorMailbox(?:.|\s){0,60}?@(%s)|/(%s)/(?:oauth2/v2\.0|B2C_1\w+|common|discovery|federationmetadata|kerberos|login|openid/|reprocess|resume|saml2|token|uxlogout|v2\.0|wsfed))`,
uuidStr,
uuidStr,
uuidStr,
Expand All @@ -33,25 +34,19 @@ var (
tenantOnMicrosoftPat = regexp.MustCompile(`([\w-]+\.onmicrosoft\.com)`)

clientIdPat = regexp.MustCompile(fmt.Sprintf(
`(?i)(?:(?:app(?:lication)?|client)(?:[ ._-]?id)?|username| -u)(?:.|\s){0,45}?(%s)`, uuidStr))
`(?i)(?:(?:api|https?)://(%s)/|myapps\.microsoft\.com/signin/(?:[\w-]+/)?(%s)|(?:[\w:=]{0,10}?(?:app(?:lication)?|cl[ie][ei]nt)(?:[ ._-]?id)?|username| -u)(?:.|\s){0,45}?(%s))`, uuidStr, uuidStr, uuidStr))
)

// FindTenantIdMatches returns a list of potential tenant IDs in the provided |data|.
func FindTenantIdMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})

for _, match := range tenantIdPat.FindAllStringSubmatch(data, -1) {
var m string
if match[1] != "" {
m = strings.ToLower(match[1])
} else if match[2] != "" {
m = strings.ToLower(match[2])
} else if match[3] != "" {
m = strings.ToLower(match[3])
} else if match[4] != "" {
m = strings.ToLower(match[4])
}
if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
m := strings.ToLower(firstNonEmptyMatch(match))

if detectors.StringShannonEntropy(m) < 3 {
continue
} else if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
continue
} else if detectors.StringShannonEntropy(m) < 3 {
continue
Expand All @@ -64,12 +59,22 @@ func FindTenantIdMatches(data string) map[string]struct{} {
return uniqueMatches
}

// language=regexp
const invalidClientPat = `(?i)(?:client[._-]?request[._-]?(?:id)?(?:.|\s){1,10}%s|cid-v1:%s)`

// FindClientIdMatches returns a list of potential client UUIDs in the provided |data|.
func FindClientIdMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})
for _, match := range clientIdPat.FindAllStringSubmatch(data, -1) {
m := strings.ToLower(match[1])
if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
m := strings.ToLower(firstNonEmptyMatch(match))

fpPat := stdRegexp.MustCompile(fmt.Sprintf(invalidClientPat, m, m))
if detectors.StringShannonEntropy(m) < 3 {
continue
} else if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
continue
} else if fpPat.MatchString(match[0]) {
// Ignore request context UUID. (https://stackoverflow.com/q/59425520)
continue
} else if detectors.StringShannonEntropy(m) < 3 {
continue
Expand Down Expand Up @@ -132,3 +137,17 @@ func queryTenant(ctx context.Context, client *http.Client, tenant string) bool {
return false
}
}

// firstNonEmptyMatch returns the index and value of the first non-empty match.
func firstNonEmptyMatch(matches []string) string {
if len(matches) <= 1 {
return ""
}
// The first index is the entire matched string.
for _, val := range matches[1:] {
if val != "" {
return val
}
}
return ""
}
72 changes: 72 additions & 0 deletions pkg/detectors/azure_entra/common_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ type testCase struct {
}

func runPatTest(t *testing.T, tests map[string]testCase, matchFunc func(data string) map[string]struct{}) {
t.Parallel()
t.Helper()
for name, test := range tests {
t.Run(name, func(t *testing.T) {
Expand Down Expand Up @@ -99,6 +100,7 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
"974fde14-c3a4-481b-9b03-cfce18213a07": {},
},
},
// https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/b444dd5f8800c298016ddd9da2e6c05b0bf4b02c/tests/Microsoft.Identity.Test.Common/TestConstants.cs#L241-L245
"login.microsoftonline.com": {
Input: ` auth: {
authority: 'https://login.microsoftonline.com/7bb339cb-e94c-4a85-884c-48ebd9bb28c3',
Expand All @@ -114,6 +116,12 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
"7bb339cb-e94c-4a85-884c-48ebd9bb28c3": {},
},
},
"login.microsoft.com": {
Input: `# SYSTEM_CONFIGURATION_ISSUER_URI=https://login.microsoft.com/2b820e29-94a2-402f-b666-c88ebcc69eb4/v2.0`,
Expected: map[string]struct{}{
"2b820e29-94a2-402f-b666-c88ebcc69eb4": {},
},
},
"sts.windows.net": {
Input: `{
"aud": "00000003-0000-0000-c000-000000000000",
Expand Down Expand Up @@ -299,6 +307,14 @@ $ApplicationId = "1e002bca-c6e2-446e-a29e-a221909fe8aa"`,
"902aeb6d-29c7-4f6e-849d-4b933117e320": {},
},
},
"cleint (typo)": {
Input: ` console.log({
cleintId:
"f3a45cb9-e388-4358-a6ef-08a63f97457c",`,
Expected: map[string]struct{}{
"f3a45cb9-e388-4358-a6ef-08a63f97457c": {},
},
},
"clientid": {
Input: `export const msalConfig = {
auth: {
Expand Down Expand Up @@ -342,6 +358,47 @@ subscription_id = "47ab1364-000d-4a53-838d-1537b1e3b49f"
"21e144ac-532d-49ad-ba15-1c40694ce8b1": {},
},
},
"uri - api://": {
Input: `AUDIENCE=api://51aaa91a-bb09-40cd-9f1f-e8c0936246c6/.default`,
Expected: map[string]struct{}{
"51aaa91a-bb09-40cd-9f1f-e8c0936246c6": {},
},
},
"uri - http://": {
Input: `AUDIENCE=http://ceb233fb-f14c-4330-9c5b-91f7db4970e1/.default`,
Expected: map[string]struct{}{
"ceb233fb-f14c-4330-9c5b-91f7db4970e1": {},
},
},
"uri - https://": {
Input: `AUDIENCE=https://47c3cfeb-b7f4-466a-b690-f7fcc79472a9/.default`,
Expected: map[string]struct{}{
"47c3cfeb-b7f4-466a-b690-f7fcc79472a9": {},
},
},
"uri - myapps.microsoft.com": {
Input: `# Linkcheck configuration
linkcheck_ignore = [
"https://myapps.microsoft.com/signin/01501f0f-c48b-4327-92a2-2324949b0a9c?tenantId=e4cbd4d7-327c-47fc-bcde-1005207021a5",
]`,
Expected: map[string]struct{}{
"01501f0f-c48b-4327-92a2-2324949b0a9c": {},
},
},
"uri - myapps.microsoft.com with name": {
Input: `$LoginURL = 'https://myapps.microsoft.com/signin/App1/c370c8f6-0cb5-44b2-a903-c6cbd5ff6ce4?tenantId=74d7c41b-e3b6-4d40-88cf-436fd5fc231a'`,
Expected: map[string]struct{}{
"c370c8f6-0cb5-44b2-a903-c6cbd5ff6ce4": {},
},
},
// TODO
// "createdBy": {
// Input: ` "systemData": {
// "createdAt": "2023-08-21T00:30:04.2907836\u002B00:00",
// "createdBy": "117311a5-df69-4fff-a301-6be98c1bd0ab",
// "createdByType": "Application"
// }`,
// },

// Arbitrary test cases
"spacing": {
Expand All @@ -356,6 +413,21 @@ subscription_id = "47ab1364-000d-4a53-838d-1537b1e3b49f"
"f12345c6-7890-1f23-b456-789eb0bb1c23": {},
},
},

// Invalid
"invalid uri": {
Input: `# ldapadd -Y EXTERNAL -H ldapi:/// -f chrootpw.ldif`,
},
"invalid - AppInsights UUID": {
Input: ` "Date": "Mon, 21 Aug 2023 00:29:56 GMT",
"Request-Context": "appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d",
"Strict-Transport-Security": "max-age=15724800; includeSubDomains; preload",`,
},
"invalid - client-request-id": {
Input: ` "Accept-Encoding": "gzip, deflate",
"client-request-id": "c9e15037-e93c-4da9-b885-9641a826ed9a",
"Connection": "keep-alive",`,
},
}

runPatTest(t, cases, FindClientIdMatches)
Expand Down
2 changes: 1 addition & 1 deletion pkg/detectors/azure_entra/serviceprincipal/sp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func VerifyCredentials(ctx context.Context, client *http.Client, tenantId string
if err := json.NewDecoder(res.Body).Decode(&errResp); err != nil {
return false, nil, err
}

switch res.StatusCode {
case http.StatusBadRequest, http.StatusUnauthorized:
// Error codes can be looked up by removing the `AADSTS` prefix.
Expand Down
45 changes: 27 additions & 18 deletions pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,13 @@ var _ interface {
detectors.Versioner
} = (*Scanner)(nil)

var (
defaultClient = common.SaneHttpClient()
// TODO: Azure storage access keys and investigate other types of creds.
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
//clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
// TODO: Tighten this regex and replace it with above.
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]`)
)
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}

func (s Scanner) Description() string {
return serviceprincipal.Description
}

func (s Scanner) Version() int {
return 1
Expand All @@ -45,6 +43,19 @@ func (s Scanner) Keywords() []string {
return []string{"azure", "az", "entra", "msal", "login.microsoftonline.com", ".onmicrosoft.com"}
}

var (
defaultClient = common.SaneHttpClient()
// TODO: Azure storage access keys and investigate other types of creds.
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
// clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
// TODO: Tighten this regex and replace it with above.
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})(?:\s|\z|[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~])`)

invalidMatchPat = regexp.MustCompile(`^passwordCredentials":`)
invalidSecretPat = regexp.MustCompile(`^[a-zA-Z]+$`)
)

// FromData will find and optionally verify Azure secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
Expand All @@ -68,20 +79,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}

func (s Scanner) Description() string {
return serviceprincipal.Description
}

func findSecretMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})
for _, match := range secretPat.FindAllStringSubmatch(data, -1) {
m := match[1]
// Ignore secrets that are handled by the V2 detector.
if v2.SecretPat.MatchString(m) {
// Ignore secrets that are handled by the V2 detector.
continue
} else if detectors.StringShannonEntropy(m) < 3 {
// Ignore low-entropy results.
continue
} else if invalidSecretPat.MatchString(m) || invalidMatchPat.MatchString(match[0]) {
// Ignore patterns that are known to be false.
continue
}
uniqueMatches[m] = struct{}{}
Expand Down
62 changes: 49 additions & 13 deletions pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,24 @@ type testCase struct {

func Test_FindClientSecretMatches(t *testing.T) {
cases := map[string]testCase{
// secret
`secret`: {
Input: `"secret": "ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~",`,
Expected: map[string]struct{}{"ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~": {}},
},

// client secret
"client_secret": {
Input: ` "TenantId": "3d7e0652-b03d-4ed2-bf86-f1299cecde17",
"ClientSecret": "gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9",`,
Expected: map[string]struct{}{"gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9": {}},
},
"client secret at end": {
Input: `secret: UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==`,
Expected: map[string]struct{}{
"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==": {},
},
},
"client_secret1": {
Input: ` public static string clientId = "413ff05b-6d54-41a7-9271-9f964bc10624";
public static string clientSecret = "k72~odcN_6TbVh5D~19_1Qkj~87trteArL";
Expand Down Expand Up @@ -72,10 +85,21 @@ configs = {"fs.azure.account.auth.type": "OAuth"`,
Input: ` "AZUREAD-AKS-APPID-SECRET": "8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-",`,
Expected: map[string]struct{}{"8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-": {}},
},
// "client_secret6": {
// Input: ``,
// Expected: map[string]struct{}{"": {}},
// },
"client_secret9": {
Input: ` client-id: 49abd816-45d1-479a-b49a-80bcf6d7213a
client-secret: 7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO`,
Expected: map[string]struct{}{"7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO": {}},
},
"client_secret10": {
Input: ` "aadClientSecret": "6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo",`,
Expected: map[string]struct{}{"6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo": {}},
},
"client_secret11": {
Input: `CLIENT_ID=9f5a8591-20b9-46b3-8317-9cc2cd52ca76
CLIENT_SECRET=2XlwmIlo3XkjE1@y[cuWsPj3_N@F23F/
TENANT_ID=08c43a10-b16a-426e-b7e8-b3b42a60e181`,
Expected: map[string]struct{}{"2XlwmIlo3XkjE1@y[cuWsPj3_N@F23F/": {}},
},

"password": {
Input: `# Login using Service Principal
Expand All @@ -86,18 +110,30 @@ $Credential = New-Object -TypeName System.Management.Automation.PSCredential -Ar
},

// False positives
"placeholder_secret": {
"invalid - placeholder_secret": {
Input: `- Log in with a service principal using a client secret:

az login --service-principal --username {{http://azure-cli-service-principal}} --password {{secret}} --tenant {{someone.onmicrosoft.com}}`,
Expected: nil,
},
// "client_secret3": {
// Input: ``,
// Expected: map[string]struct{}{
// "": {},
// },
// },
},

"invalid - only alpha characters": {
Input: `"passwordCredentials":[],"preferredTokenSigningKeyThumbprint":null,"publisherName":"Microsoft"`,
},
"invalid - low entropy": {
Input: `clientSecret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`,
},
"invalid - passwordCredentials1": {
Input: `"passwordCredentials":[{"customKeyIdentifier":"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==","endDate":"2019-07-16T23:01:19.028Z","keyId":`,
},
"invalid - passwordCredentials2": {
Input: `"passwordCredentials":[{"customKeyIdentifier":"TQB5ACAARgBpAHIAcwB0ACAASwBlAHkA"`,
},
"invalid - passwordCredentials3": {
Input: `,"passwordCredentials":[{"customKeyIdentifier":"awBlAHkAZgBvAHIAaQBtAHAAYQBsAGEA",`,
},
"invalid - azure vault path": {
Input: ` public const string MsalArlingtonOBOKeyVaultUri = "https://msidlabs.vault.azure.net:443/secrets/ARLMSIDLAB1-IDLASBS-App-CC-Secret";`,
},
}

for name, test := range cases {
Expand Down
Loading