From 26c075d3e2dbcca4031b74d2786fe160fb611a54 Mon Sep 17 00:00:00 2001 From: Deke Li <37903010+lideke@users.noreply.github.com> Date: Tue, 2 Aug 2022 14:57:25 +0200 Subject: [PATCH] Use latest version (#6) * Bump to v1.0.0, update changelog (#45) * Qa/future testing (#48) * documented reasons for untested streams * list high level scenarios for manual qa checks * remove stitch specific pr template Co-authored-by: kspeer * Add call_details stream (#49) * Add call_view core stream, filter non-attribute fields from core streams * Change `call_view` stream name to `call_details` * Make pylint happy * Fix integration tests * Remove campaign_id from foreign keys, split call_details foreign keys * Update tests to exclude call_details as needed * Add context around excluding call_details from tests * Remove outdated TODO related to addition of call_details stream. Co-authored-by: dsprayberry <28106103+dsprayberry@users.noreply.github.com> * Add core LABELS streams and campaign.labels fields to relevant reports (#53) * Add core LABELS streams and campaign.labels fields to relevant reports * Update test exclusions to exclude new core streams. * Update discover to include campaign_label in reports list * Update foreign_key expected metadata to include attributed resource foreign_keys * Update tests to re-include campaign_labels and labels as we now have test data. * Handles case where state does not have currently_syncing (#54) * Handles case where state does not have currently_syncing * tap-tester test added * takeout unused imports in test Co-authored-by: kspeer * Version bump and changelog entry (#52) * Version bump and changelog entry * Update to include PR 53. * Add PR 54 to changelog * Implement Automatic Keys (#55) * Set geographic_view.location_type as automatic to account for reporting discrepancies * Update Automatic Fields test happy / error paths * Update base to include automatic_keys metadata for use in tests * WIP Add Automatic Report Fields * Pass automatic_keys to BaseStream; use automatic_keys for inclusion * Fix bad field name in streams; Start updating tests * Rename function to match base.py * Add closing brace -_- * Fix tests; rename ad_group_criterion_criterion_id in automatic_keys; reorder test metadata; add campaign.id to campaign_audience_performance_report * Rename report_field_parts to split_report_field * Update transform_keys to raise ad_group_ad.ad fields * Update happy path streams_to_test to exclude streams with no data * Remove field name change for ad_performance_report stream * Accept Andy's Suggestion Co-authored-by: Andy Lu * Reverting Andy's change because it affects core streams * Explicitly install grpcio-status to avoid from_call attribute errors * Update setup.py with docs explaining required but unused dep Co-authored-by: Andy Lu Co-authored-by: Arthur Gorka * Cleanup (#56) * Remove useless function * Rename "REPORTS" to "STREAMS" for accuracy / readability * Version bump for PRs 56 and 55 (#57) * Version bump for PRs 56 and 55 * Update to exclude ad_group_ad change for ad_performance_report * [Feature] Add more core streams (#58) * WIP add new core streams * Set geographic_view.location_type as automatic to account for reporting discrepancies * Update Automatic Fields test happy / error paths * Update base to include automatic_keys metadata for use in tests * WIP Add Automatic Report Fields * Pass automatic_keys to BaseStream; use automatic_keys for inclusion * Fix bad field name in streams; Start updating tests * Rename function to match base.py * Add closing brace -_- * Fix tests; rename ad_group_criterion_criterion_id in automatic_keys; reorder test metadata; add campaign.id to campaign_audience_performance_report * Rename report_field_parts to split_report_field * Update transform_keys to raise ad_group_ad.ad fields * Update happy path streams_to_test to exclude streams with no data * WIP remove IPDB and try except * Fix report_definition typo and add handling for user_interest_id field * Remove field name change for ad_performance_report stream * Accept Andy's Suggestion Co-authored-by: Andy Lu * Reverting Andy's change because it affects core streams * Explicitly install grpcio-status to avoid from_call attribute errors * Remove ipdb -_- * Update setup.py with docs explaining required but unused dep * WIP w/ failing tests and attributed_resource foreign_keys * WIP with failing tests; remove extraneous attributed_resoruce automatic fields * Create UserInterestStream class to handle its edge case * Add transform_keys to UserInterestStream class; start test updates * Rename obj to json_message; rename variables accordingly; fix UserInterestStream transform_keys * update auto fields test to account for compound pks * Fix campaigns typo * Remove exclusion of feed and feed items from sync canary test Co-authored-by: Andy Lu Co-authored-by: Arthur Gorka Co-authored-by: kspeer Co-authored-by: atribed * V1.3.0 (#59) * Version bump for PRs 56 and 55 * Update to exclude ad_group_ad change for ad_performance_report * Version bump for 1.3.0 Co-authored-by: Arthur Gorka * Qa/fix build notification (#65) * fix slack notif for build * remove click_performance_report from tests * remove click_performance_report from sync canary test * run only streams that are untested in canary sync * just skip canaray test * put back the assert in the skipped test Co-authored-by: kspeer * Crest master (#66) * Tdl 19235 handle uncaught exceptions (#61) * Added backoff for 5xx, 429 and ReadTimeout errors. * Resolved pylint error. * Updated comments in the unittest cases. * Updated error handling. * TDL-18749 Implement interruptible full table streams. (#60) * Implemented interruptible full table streams. * Resolved pylint error * Resolved error in full table sync test case. * Updated config.yml to pass cci * Updated query building logic. * Updated integration test case. * Resolved review comments. * Resolved comments. * Implemeted logic to skip the duplicate records. * Resolved unittest case error. * Resolved pylint error * Resolved integration test case error * Added empty filter param for call_details and campaign_label stream. * Added unit test cases for should_sync method. * Revert "Implemeted logic to skip the duplicate records." This reverts commit cd06e11657bd35edbaefcd7f8f12acfb938e05ec. * Added logger message for debugging purpose * Updated integration test case. * Replaced .format with f string. * Updated comment in integration test. Co-authored-by: KrishnanG * Bump version (#67) Co-authored-by: KrishnanG * TDL-18524 updated readme and added sample config (#51) * updated readme and added sample config * updated endpoints * add new streams * resolved PR comments * fixed a typo * added the * Add timeout parameter to gas.search (#64) * Add timeout parameter to search gas.search * Increase timeout to 15 minutes for safety. * Add get_request_timeout function, add config to make_request as needed * Update on_giveup to raise specific exception text for timeoutexception class; add unit test * Make Pylint happy take 1 * Update make_request signature in unittests. * Another Unittest update * More unittest fixes for signature * Add default config param value for ease of implementation in future tests * Fix pylint dangerous-defaul-value * Fix stupid error * Version bump and changelog (#70) * TDL-19486 Add limit clause to core stream queries (#68) * Initial commit for add page limit. * Added limit parameter in sync method of ReportStream class. * Fixed issue for call_details stream. * Resolved unit test case error. * Added test cases. * Updated code comments. * Fixed keyerror issue. * Updated default query limit. * Updated pagination test case. * Modify config name to be more explicit. * Update comment for accuracy. * Update property name in base and propogate name change to tests * Exclude feed from streams_to_test because of lack of data * Committing stuff from main that should have already been in Co-authored-by: dsprayberry <28106103+dsprayberry@users.noreply.github.com> * Version Bump and Changelog Update (#72) * Reintroduce unintentionally removed files. (#73) * add schemaless * fix schemaless config * update login_customer_ids * fix missing key from dict * add google ads api version in config add log to show the version * use version from config * use version global Co-authored-by: bryantgray Co-authored-by: Kyle Speer <54034650+kspeer825@users.noreply.github.com> Co-authored-by: kspeer Co-authored-by: dsprayberry <28106103+dsprayberry@users.noreply.github.com> Co-authored-by: Andy Lu Co-authored-by: Arthur Gorka Co-authored-by: atribed Co-authored-by: Prijen Khokhani <88327452+prijendev@users.noreply.github.com> Co-authored-by: KrishnanG Co-authored-by: KrisPersonal <66801357+KrisPersonal@users.noreply.github.com> Co-authored-by: namrata270998 <75604662+namrata270998@users.noreply.github.com> Co-authored-by: bbaltagi-dtsl --- tap_google_ads/__init__.py | 5 ++--- tap_google_ads/streams.py | 5 +++-- tap_google_ads/sync.py | 1 + tap_google_ads/tests/base.py | 20 ++++++++++---------- tests/base.py | 12 ++++++------ 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/tap_google_ads/__init__.py b/tap_google_ads/__init__.py index 4830d9f..00f2c37 100644 --- a/tap_google_ads/__init__.py +++ b/tap_google_ads/__init__.py @@ -17,8 +17,7 @@ "refresh_token", "customer_ids", "developer_token", - "schemaless", - "version" + "schemaless" ] @@ -27,7 +26,7 @@ def main_impl(): resource_schema = create_resource_schema(args.config) state = {} - LOGGER.info(f"Sync Starting with google-ads api {args.config['version']} version.") + LOGGER.info(f"Sync Starting with google-ads api.") if args.state: state.update(args.state) if args.discover: diff --git a/tap_google_ads/streams.py b/tap_google_ads/streams.py index b7635f8..212b111 100644 --- a/tap_google_ads/streams.py +++ b/tap_google_ads/streams.py @@ -85,7 +85,9 @@ def get_selected_fields(stream_mdata): if mdata["breadcrumb"]: inclusion = mdata["metadata"].get("inclusion") selected = mdata["metadata"].get("selected") - if utils.should_sync_field(inclusion, selected) and mdata["breadcrumb"][1] != "_sdc_record_hash" and "tap-google-ads.api-field-names" in mdata["metadata"]: + if utils.should_sync_field(inclusion, selected) \ + and mdata["breadcrumb"][1] != "_sdc_record_hash" \ + and "tap-google-ads.api-field-names" in mdata["metadata"]: selected_fields.update(mdata["metadata"]["tap-google-ads.api-field-names"]) return selected_fields @@ -758,7 +760,6 @@ def sync(self, sdk_client, customer, stream, config, state, query_limit, schemal transformed_obj = self.transform_keys(json_message) record = transformer.transform(transformed_obj, stream["schema"]) record["_sdc_record_hash"] = generate_hash(record, stream_mdata) - singer.write_record(stream_name, record) else: singer.write_record(stream_name, json_message) diff --git a/tap_google_ads/sync.py b/tap_google_ads/sync.py index b32dd15..4a64fe0 100644 --- a/tap_google_ads/sync.py +++ b/tap_google_ads/sync.py @@ -72,6 +72,7 @@ def get_query_limit(config): def do_sync(config, catalog, resource_schema, state): # QA ADDED WORKAROUND [START] + try: customers = json.loads(config["customer_ids"]) except TypeError: # falling back to raw value diff --git a/tap_google_ads/tests/base.py b/tap_google_ads/tests/base.py index 30424bf..db512cd 100644 --- a/tap_google_ads/tests/base.py +++ b/tap_google_ads/tests/base.py @@ -46,14 +46,14 @@ def get_properties(self): 'end_date': '2018-04-15T00:00:00Z', 'conversion_window_days' : '-1', 'user_id': 'not used?', - 'customer_ids': os.getenv('TAP_ADWORDS_CUSTOMER_IDS')} + 'customer_ids': os.getenv('TAP_ADS_CUSTOMER_IDS')} @staticmethod def get_credentials(self): - return {'developer_token': os.getenv('TAP_ADWORDS_DEVELOPER_TOKEN'), - 'oauth_client_id': os.getenv('TAP_ADWORDS_OAUTH_CLIENT_ID'), - 'oauth_client_secret': os.getenv('TAP_ADWORDS_OAUTH_CLIENT_SECRET'), - 'refresh_token': os.getenv('TAP_ADWORDS_REFRESH_TOKEN')} + return {'developer_token': os.getenv('TAP_ADS_DEVELOPER_TOKEN'), + 'oauth_client_id': os.getenv('TAP_ADS_OAUTH_CLIENT_ID'), + 'oauth_client_secret': os.getenv('TAP_ADS_OAUTH_CLIENT_SECRET'), + 'refresh_token': os.getenv('TAP_ADS_REFRESH_TOKEN')} def expected_metadata(self): """The expected streams and metadata about the streams""" @@ -235,11 +235,11 @@ def expected_replication_method(self): def setUp(self): - missing_envs = [x for x in [os.getenv('TAP_ADWORDS_DEVELOPER_TOKEN'), - os.getenv('TAP_ADWORDS_OAUTH_CLIENT_ID'), - os.getenv('TAP_ADWORDS_OAUTH_CLIENT_SECRET'), - os.getenv('TAP_ADWORDS_REFRESH_TOKEN'), - os.getenv('TAP_ADWORDS_CUSTOMER_IDS')] if x == None] + missing_envs = [x for x in [os.getenv('TAP_ADS_DEVELOPER_TOKEN'), + os.getenv('TAP_ADS_OAUTH_CLIENT_ID'), + os.getenv('TAP_ADS_OAUTH_CLIENT_SECRET'), + os.getenv('TAP_ADS_REFRESH_TOKEN'), + os.getenv('TAP_ADS_CUSTOMER_IDS')] if x == None] if len(missing_envs) != 0: raise Exception("Missing environment variables: {}".format(missing_envs)) diff --git a/tests/base.py b/tests/base.py index 1ce4060..6ace583 100644 --- a/tests/base.py +++ b/tests/base.py @@ -58,7 +58,7 @@ def get_properties(self, original: bool = True): 'query_limit': 2, # 'conversion_window_days': '30', 'login_customer_ids': [{"customerId": os.getenv('TAP_GOOGLE_ADS_CUSTOMER_ID'), - "loginCustomerId": os.getenv('TAP_GOOGLE_ADS_LOGIN_CUSTOMER_ID'),}], + "loginCustomerId": os.getenv('TAP_GOOGLE_ADS_LOGIN_CUSTOMER_ID')}], } if original: @@ -460,11 +460,11 @@ def expected_replication_method(self): def setUp(self): - missing_envs = [x for x in [os.getenv('TAP_ADWORDS_DEVELOPER_TOKEN'), - os.getenv('TAP_ADWORDS_OAUTH_CLIENT_ID'), - os.getenv('TAP_ADWORDS_OAUTH_CLIENT_SECRET'), - os.getenv('TAP_ADWORDS_REFRESH_TOKEN'), - os.getenv('TAP_ADWORDS_CUSTOMER_IDS')] if x == None] + missing_envs = [x for x in [os.getenv('TAP_ADS_DEVELOPER_TOKEN'), + os.getenv('TAP_ADS_OAUTH_CLIENT_ID'), + os.getenv('TAP_ADS_OAUTH_CLIENT_SECRET'), + os.getenv('TAP_ADS_REFRESH_TOKEN'), + os.getenv('TAP_ADS_CUSTOMER_IDS')] if x == None] if len(missing_envs) != 0: raise Exception("Missing environment variables: {}".format(missing_envs))