Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tdl 21819 tickets stream dev changes #218

Merged
merged 19 commits into from
Feb 16, 2023
Merged

Tdl 21819 tickets stream dev changes #218

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
e758d41
created the tickets schema file
rdeshmukh15 Feb 2, 2023
12a48be
introduced new stream tickets in the hubspot tap implementation
rdeshmukh15 Feb 3, 2023
f7251dd
Changes in tickets stream implementation
rdeshmukh15 Feb 6, 2023
9dfc7d9
pylint issues fixed
rdeshmukh15 Feb 6, 2023
403e63f
pylint issues
rdeshmukh15 Feb 6, 2023
85eb770
Merge branch 'master' of github.com:singer-io/tap-hubspot into TDL-21…
rdeshmukh15 Feb 7, 2023
d5c8fcb
added unit test to test sync_tickets
rdeshmukh15 Feb 7, 2023
06b1b94
Added unit test cases in test_tickets.py
rdeshmukh15 Feb 7, 2023
7f00a83
refactoring
rdeshmukh15 Feb 7, 2023
23cd51d
added null as one of the types for the non-primary fields in tickets …
rdeshmukh15 Feb 8, 2023
f80735a
added object type in tickets.json
rdeshmukh15 Feb 8, 2023
1c9dc21
addressed PR comments
rdeshmukh15 Feb 8, 2023
828e06b
removed spaces in tickets.json
rdeshmukh15 Feb 8, 2023
0598b5c
changes in tickets.json file
rdeshmukh15 Feb 9, 2023
72818ef
added new line at the end
rdeshmukh15 Feb 9, 2023
aa4bb58
changed type from integer to string of some of the fields
rdeshmukh15 Feb 9, 2023
265b06e
add logic to transform the records
sgandhi1311 Feb 10, 2023
7c911b0
Integration tests for newly added tickets stream (#219)
sgandhi1311 Feb 15, 2023
2eacc8a
bump related changes
sgandhi1311 Feb 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ This tap:
- [Owners](http://developers.hubspot.com/docs/methods/owners/get_owners)
- [Subscription Changes](http://developers.hubspot.com/docs/methods/email/get_subscriptions_timeline)
- [Workflows](http://developers.hubspot.com/docs/methods/workflows/v3/get_workflows)
- [Tickets](https://developers.hubspot.com/docs/api/crm/tickets)
- Outputs the schema for each resource
- Incrementally pulls data based on the input state

Expand Down
73 changes: 72 additions & 1 deletion tap_hubspot/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ class StateFields:
"forms": "/forms/v2/forms",
"workflows": "/automation/v3/workflows",
"owners": "/owners/v2/owners",

"tickets": "/crm/v4/objects/tickets",
kethan1122 marked this conversation as resolved.
Show resolved Hide resolved
}

def get_start(state, tap_stream_id, bookmark_key, older_bookmark_key=None):
Expand Down Expand Up @@ -676,6 +678,7 @@ def sync_deals(STATE, ctx):
and any(prefix in breadcrumb[1] for prefix in V3_PREFIXES)]

url = get_url('deals_all')

with Transformer(UNIX_MILLISECONDS_INTEGER_DATETIME_PARSING) as bumble_bee:
for row in gen_request(STATE, 'deals', url, params, 'deals', "hasMore", ["offset"], ["offset"], v3_fields=v3_fields):
row_properties = row['properties']
Expand All @@ -699,7 +702,74 @@ def sync_deals(STATE, ctx):
singer.write_state(STATE)
return STATE

#NB> no suitable bookmark is available: https://developers.hubspot.com/docs/methods/email/get_campaigns_by_id

def gen_request_tickets(tap_stream_id, url, params, path, more_key):
"""
Cursor-based API Pagination : Used in tickets stream implementation
"""
with metrics.record_counter(tap_stream_id) as counter:
while True:
data = request(url, params).json()

if data.get(path) is None:
raise RuntimeError(
"Unexpected API response: {} not in {}".format(path, data.keys()))

for row in data[path]:
counter.increment()
yield row

if not data.get(more_key):
break
params['after'] = data.get(more_key).get('next').get('after')


def sync_tickets(STATE, ctx):
"""
Function to sync `tickets` stream records
"""
catalog = ctx.get_catalog_from_id(singer.get_currently_syncing(STATE))

stream_id = "tickets"
primary_key = "id"
bookmark_key = "updatedAt"

max_bk_value = bookmark_value = utils.strptime_with_tz(
get_start(STATE, stream_id, bookmark_key))
LOGGER.info("sync_tickets from %s", bookmark_value)

params = {'limit': 100,
'associations': 'contact,company,deals',
'archived': False
}

schema = load_schema(stream_id)
singer.write_schema(stream_id, schema, [primary_key],
[bookmark_key], catalog.get('stream_alias'))

url = get_url(stream_id)

with Transformer(UNIX_MILLISECONDS_INTEGER_DATETIME_PARSING):
for row in gen_request_tickets(stream_id, url, params, 'results', "paging"):

modified_time_org = row[bookmark_key]
modified_time = utils.strptime_with_tz(datetime.datetime.strptime(
modified_time_org, "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y-%m-%dT%H:%M:%SZ"))

# Checking the bookmark value is present on the record and it
# is greater than or equal to defined previous bookmark value
if modified_time and modified_time >= bookmark_value:
singer.write_record(stream_id, row, catalog.get(
'stream_alias'), time_extracted=utils.now())
if modified_time and modified_time >= max_bk_value:
max_bk_value = modified_time
kethan1122 marked this conversation as resolved.
Show resolved Hide resolved

STATE = singer.write_bookmark(STATE, stream_id, bookmark_key, utils.strftime(max_bk_value))
singer.write_state(STATE)
return STATE
kethan1122 marked this conversation as resolved.
Show resolved Hide resolved


# NB> no suitable bookmark is available: https://developers.hubspot.com/docs/methods/email/get_campaigns_by_id
def sync_campaigns(STATE, ctx):
catalog = ctx.get_catalog_from_id(singer.get_currently_syncing(STATE))
mdata = metadata.to_map(catalog.get('metadata'))
Expand Down Expand Up @@ -989,6 +1059,7 @@ class Stream:
Stream('contacts', sync_contacts, ["vid"], 'versionTimestamp', 'INCREMENTAL'),
Stream('deals', sync_deals, ["dealId"], 'property_hs_lastmodifieddate', 'INCREMENTAL'),
Stream('companies', sync_companies, ["companyId"], 'property_hs_lastmodifieddate', 'INCREMENTAL'),
Stream('tickets', sync_tickets, ['id'], 'updatedAt', 'INCREMENTAL'),

# Do these last as they are full table
Stream('forms', sync_forms, ['guid'], 'updatedAt', 'FULL_TABLE'),
Expand Down
196 changes: 196 additions & 0 deletions tap_hubspot/schemas/tickets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
{
"type": "object",
"properties": {
"id": {
"type": "integer"
},
"createdAt": {
"type": [
"null",
"string"
],
"format": "date-time"
},
"updatedAt": {
"type": [
"null",
"string"
],
"format": "date-time"
},
"archived": {
"type": [
"null",
"boolean"
]
},
"properties": {
"type": [
"null",
"object"
],
"properties": {
"content": {
"type": [
"null",
"string"
]
},
"createdate": {
"type": [
"null",
"string"
],
"format": "date-time"
},
"hs_lastmodifieddate": {
"type": [
"null",
"string"
],
"format": "date-time"
},
"hs_object_id": {
"type": [
"null",
"integer"
]
},
"hs_pipeline": {
"type": [
"null",
"integer"
]
},
"hs_pipeline_stage": {
"type": [
"null",
"integer"
]
},
"hs_ticket_category": {
"type": [
"null",
"string"
]
},
"hs_ticket_priority": {
"type": [
"null",
"string"
]
},
"subject": {
"type": [
"null",
"string"
]
}
}
},
"associations": {
"type": [
"null",
"object"
],
"properties": {
"companies": {
"type": [
"null",
"object"
],
"properties": {
"results": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"properties": {
"id": {
"type": [
"null",
"integer"
]
},
"type": {
"type": [
"null",
"string"
]
}
}
}
}
},
"deals": {
"type": [
"null",
"object"
],
"properties": {
"results": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"properties": {
"id": {
"type": [
"null",
"integer"
]
},
"type": {
"type": [
"null",
"string"
]
}
}
}
}
},
"contacts": {
"type": [
"null",
"object"
],
"properties": {
"results": [
"null",
"array"
],
"items": {
"type": [
"null",
"object"
],
"properties": {
"id": {
"type": [
"null",
"integer"
]
},
"type": {
"type": [
"null",
"string"
]
}
}
}
}
}
}
}
}
}
Loading