Skip to content

Commit

Permalink
fix(ingest-idp): emit empty GroupMembership when there are no groups
Browse files Browse the repository at this point in the history
  • Loading branch information
aditya-radhakrishnan committed Jan 31, 2023
1 parent efaf092 commit 83c5fec
Show file tree
Hide file tree
Showing 5 changed files with 388 additions and 3 deletions.
5 changes: 4 additions & 1 deletion metadata-ingestion/src/datahub/ingestion/source/identity/okta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import logging
import re
import urllib
from collections import defaultdict
from dataclasses import dataclass, field
from time import sleep
from typing import Dict, Iterable, List, Optional, Union
Expand Down Expand Up @@ -316,7 +317,9 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
yield group_status_wu

# Step 2: Populate GroupMembership Aspects for CorpUsers
datahub_corp_user_urn_to_group_membership: Dict[str, GroupMembershipClass] = {}
datahub_corp_user_urn_to_group_membership: Dict[
str, GroupMembershipClass
] = defaultdict(lambda: GroupMembershipClass(groups=[]))
if self.config.ingest_group_membership and okta_groups is not None:
# Fetch membership for each group.
for okta_group in okta_groups:
Expand Down
3 changes: 1 addition & 2 deletions metadata-ingestion/src/datahub/ingestion/source/ldap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,8 +400,7 @@ def build_corp_user_mce(
],
)

if groups:
user_snapshot.aspects.append(GroupMembershipClass(groups=groups))
user_snapshot.aspects.append(GroupMembershipClass(groups=groups))

return MetadataChangeEvent(proposedSnapshot=user_snapshot)

Expand Down
34 changes: 34 additions & 0 deletions metadata-ingestion/tests/integration/azure_ad/azure_ad_groups.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,5 +66,39 @@
"theme": null,
"visibility": null,
"onPremisesProvisioningErrors": []
},
{
"id": "00000000-0000-0000-0000-0000000000002",
"deletedDateTime": null,
"classification": null,
"createdDateTime": "2021-08-20 11: 00: 00",
"creationOptions": [],
"description": "This is an interesting description",
"displayName": "groupDisplayName3",
"expirationDateTime": null,
"groupTypes": [],
"isAssignableToRole": null,
"mail": "[email protected]",
"mailEnabled": false,
"mailNickname": "groupDisplayName3",
"membershipRule": null,
"membershipRuleProcessingState": null,
"onPremisesDomainName": null,
"onPremisesLastSyncDateTime": null,
"onPremisesNetBiosName": null,
"onPremisesSamAccountName": null,
"onPremisesSecurityIdentifier": null,
"onPremisesSyncEnabled": null,
"preferredDataLocation": null,
"preferredLanguage": null,
"proxyAddresses": [],
"renewedDateTime": "2021-08-20 11:00:00",
"resourceBehaviorOptions": [],
"resourceProvisioningOptions": [],
"securityEnabled": true,
"securityIdentifier": "xxxxx",
"theme": null,
"visibility": null,
"onPremisesProvisioningErrors": []
}
]
292 changes: 292 additions & 0 deletions ...ata-ingestion/tests/integration/azure_ad/azure_ad_mces_golden_empty_group_membership.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,292 @@
[
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": {
"urn": "urn:li:corpGroup:groupDisplayName1",
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"displayName": "groupDisplayName1",
"email": null,
"admins": [],
"members": [],
"groups": [],
"description": null,
"slack": null
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "corpGroup",
"entityUrn": "urn:li:corpGroup:groupDisplayName1",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "origin",
"aspect": {
"value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "corpGroup",
"entityUrn": "urn:li:corpGroup:groupDisplayName1",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"value": "{\"removed\": false}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": {
"urn": "urn:li:corpGroup:groupDisplayName2",
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"displayName": "groupDisplayName2",
"email": "[email protected]",
"admins": [],
"members": [],
"groups": [],
"description": "This is an interesting description",
"slack": null
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "corpGroup",
"entityUrn": "urn:li:corpGroup:groupDisplayName2",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "origin",
"aspect": {
"value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "corpGroup",
"entityUrn": "urn:li:corpGroup:groupDisplayName2",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"value": "{\"removed\": false}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": {
"urn": "urn:li:corpuser:[email protected]",
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpUserInfo": {
"active": true,
"displayName": "John Green",
"email": "[email protected]",
"title": null,
"managerUrn": null,
"departmentId": null,
"departmentName": null,
"firstName": "John",
"lastName": "Green",
"fullName": "John Green",
"countryCode": null
}
},
{
"com.linkedin.pegasus2avro.identity.GroupMembership": {
"groups": [
"urn:li:corpGroup:groupDisplayName1",
"urn:li:corpGroup:groupDisplayName2"
]
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "origin",
"aspect": {
"value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"value": "{\"removed\": false}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": {
"urn": "urn:li:corpuser:[email protected]",
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpUserInfo": {
"active": true,
"displayName": "Adam Hall",
"email": "[email protected]",
"title": null,
"managerUrn": null,
"departmentId": null,
"departmentName": null,
"firstName": "Adam",
"lastName": "Hall",
"fullName": "Adam Hall",
"countryCode": null
}
},
{
"com.linkedin.pegasus2avro.identity.GroupMembership": {
"groups": [
]
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "origin",
"aspect": {
"value": "{\"type\": \"EXTERNAL\", \"externalType\": \"AZURE_AD\"}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
},
{
"auditHeader": null,
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"entityKeyAspect": null,
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"value": "{\"removed\": false}",
"contentType": "application/json"
},
"systemMetadata": {
"lastObserved": 1629795600000,
"runId": "test-azure-ad",
"registryName": null,
"registryVersion": null,
"properties": null
}
}
]
Loading

0 comments on commit 83c5fec

Please sign in to comment.