diff --git a/.circleci/config.yml b/.circleci/config.yml index 988bb78..de55bc9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,50 +2,123 @@ version: 2.1 orbs: slack: circleci/slack@3.4.2 -jobs: - build: +executors: + docker-executor: docker: - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester + +jobs: + # TODO remove if not needed + # build: + # executor: docker-executor + # steps: + # - run: echo 'CI done' + ensure_env: + executor: docker-executor steps: - checkout - run: name: 'Setup virtual env' command: | - aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh python3 -mvenv /usr/local/share/virtualenvs/tap-google-ads source /usr/local/share/virtualenvs/tap-google-ads/bin/activate - pip install -U pip setuptools - pip install -e .[dev] + pip install 'pip==21.1.3' + pip install 'setuptools==56.0.0' + pip install .[dev] + - slack/notify-on-failure: + only_for_branches: master + - persist_to_workspace: + root: /usr/local/share/virtualenvs + paths: + - tap-google-ads + run_pylint: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs + - run: + name: 'Run pylint' + command: | + source /usr/local/share/virtualenvs/tap-google-ads/bin/activate + aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh + source dev_env.sh + echo "$PYLINT_DISABLE_LIST" + pylint tap_google_ads --disable "$PYLINT_DISABLE_LIST" + - slack/notify-on-failure: + only_for_branches: master + + run_unit_tests: + executor: docker-executor + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: - name: 'pylint' + name: 'Run Unit Tests' command: | source /usr/local/share/virtualenvs/tap-google-ads/bin/activate - # TODO: Adjust the pylint disables - pylint tap_google_ads --disable 'broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,bad-whitespace,missing-class-docstring' - # TODO implement this run block when tests are avialable! - # - run: - # name: 'Unit Tests' - # command: | - # source /usr/local/share/virtualenvs/tap-google-ads/bin/activate - # nosetests tests/unittests + pip install nose coverage + nosetests --with-coverage --cover-erase --cover-package=tap_google_ads --cover-html-dir=htmlcov tests/unittests + coverage html + - store_test_results: + path: test_output/report.xml + - store_artifacts: + path: htmlcov + - slack/notify-on-failure: + only_for_branches: master + + run_integration_tests: + executor: docker-executor + parallelism: 6 + steps: + - checkout + - attach_workspace: + at: /usr/local/share/virtualenvs - run: - name: 'Integration Tests' + name: 'Run Integration Tests' command: | + aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh source dev_env.sh source /usr/local/share/virtualenvs/tap-tester/bin/activate - run-test --tap=tap-google-ads tests + circleci tests glob "tests/*.py" | circleci tests split > ./tests-to-run + if [ -s ./tests-to-run ]; then + for test_file in $(cat ./tests-to-run) + do + run-test --tap=${CIRCLE_PROJECT_REPONAME} $test_file + done + fi - slack/notify-on-failure: - only_for_branches: main + only_for_branches: master workflows: version: 2 - commit: + commit: &commit_jobs jobs: - - build: + - ensure_env: + context: + - circleci-user + - tier-1-tap-user + - run_pylint: context: - circleci-user - - tap-tester-user + - tier-1-tap-user + requires: + - ensure_env + - run_unit_tests: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env + - run_integration_tests: + context: + - circleci-user + - tier-1-tap-user + requires: + - ensure_env build_daily: + <<: *commit_jobs triggers: - schedule: cron: "0 3 * * *" @@ -53,8 +126,3 @@ workflows: branches: only: - main - jobs: - - build: - context: - - circleci-user - - tap-tester-user diff --git a/spikes/schema_gen_protobuf.py b/spikes/schema_gen_protobuf.py deleted file mode 100644 index 0e076cc..0000000 --- a/spikes/schema_gen_protobuf.py +++ /dev/null @@ -1,159 +0,0 @@ -import importlib -import json -import os -import pkgutil -import re -from google.ads.googleads.v9.resources.types import campaign, ad, ad_group, customer -from google.protobuf.pyext.cpp_message import GeneratedProtocolMessageType -from google.protobuf.pyext._message import RepeatedScalarContainer, RepeatedCompositeContainer - -#>>> type(campaign.Campaign()._pb.target_spend.__class__) -# - -# Unknown types lookup to their actual class in the code. For some reason these differ. -# Unknown classes should be found in this project, likely somehwere around here: -# https://github.com/googleads/google-ads-python/tree/14.1.0/google/ads/googleads/v9/common/types -type_lookup = {"google.ads.googleads.v9.common.FinalAppUrl": "google.ads.googleads.v9.common.types.final_app_url.FinalAppUrl", - "google.ads.googleads.v9.common.AdVideoAsset": "google.ads.googleads.v9.common.types.ad_asset.AdVideoAsset", - "google.ads.googleads.v9.common.AdTextAsset": "google.ads.googleads.v9.common.types.ad_asset.AdTextAsset", - "google.ads.googleads.v9.common.AdMediaBundleAsset": "google.ads.googleads.v9.common.types.ad_asset.AdMediaBundleAsset", - "google.ads.googleads.v9.common.AdImageAsset": "google.ads.googleads.v9.common.types.ad_asset.AdImageAsset", - - "google.ads.googleads.v9.common.PolicyTopicEntry": "google.ads.googleads.v9.common.types.policy.PolicyTopicEntry", - "google.ads.googleads.v9.common.PolicyTopicConstraint": "google.ads.googleads.v9.common.types.policy.PolicyTopicConstraint", - "google.ads.googleads.v9.common.PolicyTopicEvidence": "google.ads.googleads.v9.common.types.policy.PolicyTopicEvidence", - "google.ads.googleads.v9.common.PolicyTopicConstraint.CountryConstraint": "google.ads.googleads.v9.common.types.policy.PolicyTopicConstraint", # This one's weird, handling it manually in the generator - - "google.ads.googleads.v9.common.UrlCollection": "google.ads.googleads.v9.common.types.url_collection.UrlCollection", - - "google.ads.googleads.v9.common.CustomParameter": "google.ads.googleads.v9.common.types.custom_parameter.CustomParameter", - - "google.ads.googleads.v9.common.ProductImage": "google.ads.googleads.v9.common.types.ad_type_infos.ProductImage", - "google.ads.googleads.v9.common.ProductVideo": "google.ads.googleads.v9.common.types.ad_type_infos.ProductVideo", - - "google.ads.googleads.v9.common.FrequencyCapEntry": "google.ads.googleads.v9.common.types.frequency_cap.FrequencyCapEntry", - - "google.ads.googleads.v9.common.TargetRestriction": "google.ads.googleads.v9.common.types.targeting_setting.TargetRestriction", - "google.ads.googleads.v9.common.TargetRestrictionOperation": "google.ads.googleads.v9.common.types.targeting_setting.TargetRestrictionOperation", - } - -# From: https://stackoverflow.com/questions/19053707/converting-snake-case-to-lower-camel-case-lowercamelcase -def to_camel_case(snake_str): - components = snake_str.split('_') - # We capitalize the first letter of each component except the first one - # with the 'title' method and join them together. - return components[0] + ''.join(x.title() for x in components[1:]) - -def type_to_json_schema(typ): - # TODO: Bytes in an anyOf gives us, usually, just 'string', so it can be a non-anyOf? - if typ == 'bytes': - return {"type": ["null","UNSUPPORTED_string"]} - elif typ == 'int': - return {"type": ["null","integer"]} - elif typ in ['str','unicode']: - return {"type": ["null","string"]} - elif typ == 'long': - return {"type": ["null","integer"]} - else: - raise Exception(f"Unknown scalar type {typ}") - -def handle_scalar_container(acc, prop_val, prop_camel): - try: - prop_val.append(1) - prop_val.append(True) - prop_val.append(0.0) - except TypeError as e: - re_result = re.search(r"but expected one of: (.+)$", str(e)) - if re_result: - actual_types = re_result.groups()[0].split(',') - actual_types = [t.strip() for t in actual_types] - acc[prop_camel] = {"type": ["null", "array"], - "items": {"anyOf": [type_to_json_schema(t) for t in actual_types]}} - else: - raise - -ref_schema_lookup = {} -def handle_composite_container(acc, prop_val, prop_camel): - try: - prop_val.append(1) - prop_val.append(True) - prop_val.append(0.0) - except TypeError as e: - re_result = re.search(r"expected (.+) got ", str(e)) - if not re_result: - import ipdb; ipdb.set_trace() - 1+1 - raise - shown_type = re_result.groups()[0] - actual_type = type_lookup.get(shown_type) - if not actual_type: - print(f"Unknown composite type: {shown_type}") - else: - # TODO: Should we just build the objects based on the type name and insert them as a definition and $ref to save space? - mod = importlib.import_module('.'.join(actual_type.split('.')[:-1])) - if shown_type == "google.ads.googleads.v9.common.PolicyTopicConstraint.CountryConstraint": - obj = getattr(mod, actual_type.split('.')[-1]).CountryConstraint() - else: - obj = getattr(mod, actual_type.split('.')[-1])() - type_name = shown_type.split('.')[-1] - acc[prop_camel] = {"type": ["null", "array"], - "items":{"$ref": f"#/definitions/{type_name}"}} - if type_name not in ref_schema_lookup: - ref_schema_lookup[type_name] = get_schema({},obj._pb) - -def get_schema(acc, current): - for prop in filter(lambda p: re.search(r"^[a-z]", p), dir(current)): - try: - prop_val = getattr(current, prop) - prop_camel = to_camel_case(prop) - if isinstance(prop_val.__class__, GeneratedProtocolMessageType): - # TODO: Should we just build the objects based on the type name and insert them as a definition and $ref to save space? - new_acc_obj = {} - type_name = type(prop_val).__qualname__ - acc[prop_camel] = {"$ref": f"#/definitions/{type_name}"} - if type_name not in ref_schema_lookup: - ref_schema_lookup[type_name] = {"type": ["null", "object"], - "properties": get_schema(new_acc_obj, prop_val)} - elif isinstance(prop_val, bool): - acc[prop_camel] = {"type": ["null", "boolean"]} - elif isinstance(prop_val, str): - acc[prop_camel] = {"type": ["null", "string"]} - elif isinstance(prop_val, int): - acc[prop_camel] = {"type": ["null", "integer"]} - elif isinstance(prop_val, float): - acc[prop_camel] = {"type": ["null", "string"], - "format": "singer.decimal"} - elif isinstance(prop_val, bytes): - # TODO: Should this just be empty? Then put it elsewhere to mark as unsupported? With a message? - # - Or should we just make it string? - acc[prop_camel] = {"type": ["null", "UNSUPPORTED_string"]} - elif isinstance(prop_val, RepeatedScalarContainer): - handle_scalar_container(acc, prop_val, prop_camel) - elif isinstance(prop_val, RepeatedCompositeContainer): - handle_composite_container(acc, prop_val, prop_camel) - else: - import ipdb; ipdb.set_trace() - 1+1 - raise Exception(f"Unhandled type {type(prop_val)}") - except Exception as e: - raise - #import ipdb; ipdb.set_trace() - # 1+1 - return acc - -def root_get_schema(obj, pb): - schema = get_schema(obj, pb) - global ref_schema_lookup - schema["definitions"] = ref_schema_lookup - ref_schema_lookup = {} - return schema - -with open("auto_campaign.json", "w") as f: - json.dump(root_get_schema({}, campaign.Campaign()._pb), f) -with open("auto_ad.json", "w") as f: - json.dump(root_get_schema({}, ad.Ad()._pb), f) -with open("auto_ad_group.json", "w") as f: - json.dump(root_get_schema({}, ad_group.AdGroup()._pb), f) -with open("auto_account.json", "w") as f: - json.dump(root_get_schema({}, customer.Customer()._pb), f) -print("Wrote schemas to local directory under auto_*.json, please review and manually set datetime formats on datetime fields and change Enum field types to 'string' schema.") diff --git a/tap_google_ads/__init__.py b/tap_google_ads/__init__.py index 5775d5b..307a96d 100644 --- a/tap_google_ads/__init__.py +++ b/tap_google_ads/__init__.py @@ -1,23 +1,11 @@ #!/usr/bin/env python3 -import json -import os -import re -import sys - import singer from singer import utils -from singer import bookmarks -from singer import metadata -from singer import transform, UNIX_MILLISECONDS_INTEGER_DATETIME_PARSING, Transformer - -from google.ads.googleads.client import GoogleAdsClient -from google.ads.googleads.errors import GoogleAdsException -from google.protobuf.json_format import MessageToJson -from tap_google_ads.reports import initialize_core_streams -from tap_google_ads.reports import initialize_reports +from tap_google_ads.discover import create_resource_schema +from tap_google_ads.discover import do_discover +from tap_google_ads.sync import do_sync -API_VERSION = "v9" LOGGER = singer.get_logger() @@ -30,402 +18,34 @@ "developer_token", ] -CORE_ENDPOINT_MAPPINGS = { - "campaign": {"primary_keys": ["id"], "stream_name": "campaigns"}, - "ad_group": {"primary_keys": ["id"], "stream_name": "ad_groups"}, - "ad_group_ad": {"primary_keys": ["id"], "stream_name": "ads"}, - "customer": {"primary_keys": ["id"], "stream_name": "accounts"}, -} - -REPORTS = [ - "accessible_bidding_strategy", - "ad_group", - "ad_group_ad", - "ad_group_audience_view", - "age_range_view", - "bidding_strategy", - "call_view", - "campaign", - "campaign_audience_view", - "campaign_budget", - "click_view", - "customer", - "display_keyword_view", - "dynamic_search_ads_search_term_view", - "expanded_landing_page_view", - "feed_item", - "feed_item_target", - "feed_placeholder_view", - "gender_view", - "geographic_view", - "keyword_view", - "landing_page_view", - "managed_placement_view", - "search_term_view", - "shopping_performance_view", - "topic_view", - "user_location_view", - "video", -] - -CATEGORY_MAP = { - 0: "UNSPECIFIED", - 1: "UNKNOWN", - 2: "RESOURCE", - 3: "ATTRIBUTE", - 5: "SEGMENT", - 6: "METRIC", -} - - -def get_attributes(api_objects, resource): - resource_attributes = [] - - if CATEGORY_MAP[resource.category] != "RESOURCE": - # Attributes, segments, and metrics do not have attributes - return resource_attributes - - attributed_resources = set(resource.attribute_resources) - for field in api_objects: - root_object_name = field.name.split(".")[0] - does_field_exist_on_resource = ( - root_object_name == resource.name - or root_object_name in attributed_resources - ) - is_field_an_attribute = CATEGORY_MAP[field.category] == "ATTRIBUTE" - if is_field_an_attribute and does_field_exist_on_resource: - resource_attributes.append(field.name) - return resource_attributes - - -def get_segments(resource_schema, resource): - resource_segments = [] - - if resource["category"] != "RESOURCE": - # Attributes, segments, and metrics do not have attributes - return resource_segments - - segments = resource["segments"] - for segment in segments: - if segment.startswith("segments."): - resource_segments.append(segment) - else: - segment_schema = resource_schema[segment] - segment_attributes = [ - attribute - for attribute in segment_schema["attributes"] - if attribute.startswith(f"{segment}.") - ] - resource_segments.extend(segment_attributes) - return resource_segments - - -def create_resource_schema(config): - client = GoogleAdsClient.load_from_dict(get_client_config(config)) - gaf_service = client.get_service("GoogleAdsFieldService") - - query = "SELECT name, category, data_type, selectable, filterable, sortable, selectable_with, metrics, segments, is_repeated, type_url, enum_values, attribute_resources" - - api_objects = gaf_service.search_google_ads_fields(query=query) - - # These are the data types returned from google. They are mapped to json schema. UNSPECIFIED and UNKNOWN have never been returned. - # 0: "UNSPECIFIED", 1: "UNKNOWN", 2: "BOOLEAN", 3: "DATE", 4: "DOUBLE", 5: "ENUM", 6: "FLOAT", 7: "INT32", 8: "INT64", 9: "MESSAGE", 10: "RESOURCE_NAME", 11: "STRING", 12: "UINT64" - data_type_map = { - 0: {"type": ["null", "string"]}, - 1: {"type": ["null", "string"]}, - 2: {"type": ["null", "boolean"]}, - 3: {"type": ["null", "string"]}, - 4: {"type": ["null", "string"], "format": "singer.decimal"}, - 5: {"type": ["null", "string"]}, - 6: {"type": ["null", "string"], "format": "singer.decimal"}, - 7: {"type": ["null", "integer"]}, - 8: {"type": ["null", "integer"]}, - 9: {"type": ["null", "object", "string"], "properties": {}}, - 10: {"type": ["null", "object", "string"], "properties": {}}, - 11: {"type": ["null", "string"]}, - 12: {"type": ["null", "integer"]}, - } - - resource_schema = {} - - for resource in api_objects: - attributes = get_attributes(api_objects, resource) - - resource_metadata = { - "name": resource.name, - "category": CATEGORY_MAP[resource.category], - "json_schema": data_type_map[resource.data_type], - "selectable": resource.selectable, - "filterable": resource.filterable, - "sortable": resource.sortable, - "selectable_with": set(resource.selectable_with), - "metrics": list(resource.metrics), - "segments": list(resource.segments), - "attributes": attributes, - } - - resource_schema[resource.name] = resource_metadata - - for resource in resource_schema.values(): - updated_segments = get_segments(resource_schema, resource) - resource["segments"] = updated_segments - - for report in REPORTS: - report_object = resource_schema[report] - fields = {} - attributes = report_object["attributes"] - metrics = report_object["metrics"] - segments = report_object["segments"] - for field in attributes + metrics + segments: - field_schema = dict(resource_schema[field]) - - if field_schema["name"] in segments: - field_schema["category"] = "SEGMENT" - - fields[field_schema["name"]] = { - "field_details": field_schema, - "incompatible_fields": [], - } - - metrics_and_segments = set(metrics + segments) - for field_name, field in fields.items(): - if field["field_details"]["category"] == "ATTRIBUTE": - continue - for compared_field in metrics_and_segments: - - if not ( - field_name.startswith("segments.") - or field_name.startswith("metrics.") - ): - field_root_resource = field_name.split(".")[0] - else: - field_root_resource = None - - if (field_name != compared_field) and ( - compared_field.startswith("metrics.") - or compared_field.startswith("segments.") - ): - field_to_check = field_root_resource or field_name - if ( - field_to_check - not in resource_schema[compared_field]["selectable_with"] - ): - field["incompatible_fields"].append(compared_field) - - report_object["fields"] = fields - return resource_schema - - -def canonicalize_name(name): - """Remove all dot and underscores and camel case the name.""" - tokens = re.split("\\.|_", name) - - first_word = [tokens[0]] - other_words = [word.capitalize() for word in tokens[1:]] - - return "".join(first_word + other_words) - - -def do_discover_core_streams(resource_schema): - adwords_to_google_ads = initialize_core_streams(resource_schema) - - catalog = [] - for stream_name, stream in adwords_to_google_ads.items(): - resource_object = resource_schema[stream.google_ads_resources_name[0]] - fields = resource_object["fields"] - report_schema = {} - report_metadata = { - tuple(): { - "inclusion": "available", - "table-key-properties": stream.primary_keys, - } - } - - for field, props in fields.items(): - resource_matches = field.startswith(resource_object["name"] + ".") - is_id_field = field.endswith(".id") - - if props["field_details"]["category"] == "ATTRIBUTE" and ( - resource_matches or is_id_field - ): - if resource_matches: - field = ".".join(field.split(".")[1:]) - elif is_id_field: - field = field.replace(".", "_") - - the_schema = props["field_details"]["json_schema"] - report_schema[field] = the_schema - report_metadata[("properties", field)] = { - "fieldExclusions": props["incompatible_fields"], - "behavior": props["field_details"]["category"], - } - if field in stream.primary_keys: - inclusion = "automatic" - elif props["field_details"]["selectable"]: - inclusion = "available" - else: - inclusion = "unsupported" - report_metadata[("properties", field)]["inclusion"] = inclusion - catalog_entry = { - "tap_stream_id": stream.google_ads_resources_name[0], - "stream": stream_name, - "schema": { - "type": ["null", "object"], - "properties": report_schema, - }, - "metadata": singer.metadata.to_list(report_metadata), - } - catalog.append(catalog_entry) - return catalog - - -def create_field_metadata(primary_key, schema): - mdata = {} - mdata = metadata.write(mdata, (), "inclusion", "available") - mdata = metadata.write(mdata, (), "table-key-properties", primary_key) - - for field in schema["properties"]: - breadcrumb = ("properties", str(field)) - mdata = metadata.write(mdata, breadcrumb, "inclusion", "available") - - mdata = metadata.write( - mdata, ("properties", primary_key[0]), "inclusion", "automatic" - ) - mdata = metadata.to_list(mdata) - - return mdata - - -def create_sdk_client(config, login_customer_id=None): - CONFIG = { - "use_proto_plus": False, - "developer_token": config["developer_token"], - "client_id": config["oauth_client_id"], - "client_secret": config["oauth_client_secret"], - # "access_token": config["access_token"], # BUG? REMOVE ME! - "refresh_token": config["refresh_token"], - } - - if login_customer_id: - CONFIG["login_customer_id"] = login_customer_id - - sdk_client = GoogleAdsClient.load_from_dict(CONFIG) - return sdk_client - - -def do_sync(config, catalog, resource_schema): - # QA ADDED WORKAROUND [START] - try: - customers = json.loads(config["login_customer_ids"]) - except TypeError: # falling back to raw value - customers = config["login_customer_ids"] - # QA ADDED WORKAROUND [END] - - selected_streams = [ - stream - for stream in catalog["streams"] - if singer.metadata.to_map(stream["metadata"])[()].get("selected") - ] - - core_streams = initialize_core_streams(resource_schema) - - for customer in customers: - sdk_client = create_sdk_client(config, customer["loginCustomerId"]) - for catalog_entry in selected_streams: - stream_name = catalog_entry["stream"] - if stream_name in core_streams: - stream_obj = core_streams[stream_name] - - mdata_map = singer.metadata.to_map(catalog_entry["metadata"]) - - primary_key = ( - mdata_map[()].get("metadata", {}).get("table-key-properties", []) - ) - singer.messages.write_schema( - stream_name, catalog_entry["schema"], primary_key - ) - stream_obj.sync(sdk_client, customer, catalog_entry) - - -def do_discover(resource_schema): - core_streams = do_discover_core_streams(resource_schema) - # report_streams = do_discover_reports(resource_schema) - streams = [] - streams.extend(core_streams) - # streams.extend(report_streams) - json.dump({"streams": streams}, sys.stdout, indent=2) - - -def do_discover_reports(resource_schema): - ADWORDS_TO_GOOGLE_ADS = initialize_reports(resource_schema) - - streams = [] - for adwords_report_name, report in ADWORDS_TO_GOOGLE_ADS.items(): - report_mdata = {tuple(): {"inclusion": "available"}} - try: - for report_field in report.fields: - # field = resource_schema[report_field] - report_mdata[("properties", report_field)] = { - # "fieldExclusions": report.field_exclusions.get(report_field, []), - # "behavior": report.behavior.get(report_field, "ATTRIBUTE"), - "fieldExclusions": report.field_exclusions[report_field], - "behavior": report.behavior[report_field], - } - - if report.behavior[report_field]: - inclusion = "available" - else: - inclusion = "unsupported" - report_mdata[("properties", report_field)]["inclusion"] = inclusion - except Exception as err: - print(f"Error in {adwords_report_name}") - raise err - - catalog_entry = { - "tap_stream_id": adwords_report_name, - "stream": adwords_report_name, - "schema": { - "type": ["null", "object"], - "is_report": True, - "properties": report.schema, - }, - "metadata": singer.metadata.to_list(report_mdata), - } - streams.append(catalog_entry) - - return streams - - -def get_client_config(config, login_customer_id=None): - client_config = { - "use_proto_plus": False, - "developer_token": config["developer_token"], - "client_id": config["oauth_client_id"], - "client_secret": config["oauth_client_secret"], - "refresh_token": config["refresh_token"], - # "access_token": config["access_token"], # BUG? REMOVE ME - } - - if login_customer_id: - client_config["login_customer_id"] = login_customer_id - - return client_config - - -def main(): +def main_impl(): args = utils.parse_args(REQUIRED_CONFIG_KEYS) - resource_schema = create_resource_schema(args.config) + state = {} + + if args.state: + state.update(args.state) if args.discover: do_discover(resource_schema) LOGGER.info("Discovery complete") elif args.catalog: - do_sync(args.config, args.catalog.to_dict(), resource_schema) + do_sync(args.config, args.catalog.to_dict(), resource_schema, state) LOGGER.info("Sync Completed") else: LOGGER.info("No properties were selected") +def main(): + + try: + main_impl() + except Exception as e: + LOGGER.exception(e) + for line in str(e).splitlines(): + LOGGER.critical(line) + raise e + + if __name__ == "__main__": main() diff --git a/tap_google_ads/client.py b/tap_google_ads/client.py new file mode 100644 index 0000000..666083e --- /dev/null +++ b/tap_google_ads/client.py @@ -0,0 +1,17 @@ +from google.ads.googleads.client import GoogleAdsClient + + +def create_sdk_client(config, login_customer_id=None): + CONFIG = { + "use_proto_plus": False, + "developer_token": config["developer_token"], + "client_id": config["oauth_client_id"], + "client_secret": config["oauth_client_secret"], + "refresh_token": config["refresh_token"], + } + + if login_customer_id: + CONFIG["login_customer_id"] = login_customer_id + + sdk_client = GoogleAdsClient.load_from_dict(CONFIG) + return sdk_client diff --git a/tap_google_ads/discover.py b/tap_google_ads/discover.py new file mode 100644 index 0000000..04df284 --- /dev/null +++ b/tap_google_ads/discover.py @@ -0,0 +1,254 @@ +import json +import sys + +import singer + +from tap_google_ads.client import create_sdk_client +from tap_google_ads.streams import initialize_core_streams +from tap_google_ads.streams import initialize_reports + +API_VERSION = "v9" + +LOGGER = singer.get_logger() + +REPORTS = [ + "accessible_bidding_strategy", + "ad_group", + "ad_group_ad", + "ad_group_audience_view", + "age_range_view", + "bidding_strategy", + "call_view", + "campaign", + "campaign_audience_view", + "campaign_budget", + "campaign_criterion", + "click_view", + "customer", + "display_keyword_view", + "dynamic_search_ads_search_term_view", + "expanded_landing_page_view", + "feed_item", + "feed_item_target", + "feed_placeholder_view", + "gender_view", + "geographic_view", + "keyword_view", + "landing_page_view", + "managed_placement_view", + "search_term_view", + "shopping_performance_view", + "topic_view", + "user_location_view", + "video", +] + +CATEGORY_MAP = { + 0: "UNSPECIFIED", + 1: "UNKNOWN", + 2: "RESOURCE", + 3: "ATTRIBUTE", + 5: "SEGMENT", + 6: "METRIC", +} + + +def get_api_objects(config): + client = create_sdk_client(config) + gaf_service = client.get_service("GoogleAdsFieldService") + + query = "SELECT name, category, data_type, selectable, filterable, sortable, selectable_with, metrics, segments, is_repeated, type_url, enum_values, attribute_resources" + + api_objects = gaf_service.search_google_ads_fields(query=query) + return api_objects + + +def get_attributes(api_objects, resource): + resource_attributes = [] + + if CATEGORY_MAP[resource.category] != "RESOURCE": + # Attributes, segments, and metrics do not have attributes + return resource_attributes + + attributed_resources = set(resource.attribute_resources) + for field in api_objects: + root_object_name = field.name.split(".")[0] + does_field_exist_on_resource = ( + root_object_name == resource.name + or root_object_name in attributed_resources + ) + is_field_an_attribute = CATEGORY_MAP[field.category] == "ATTRIBUTE" + if is_field_an_attribute and does_field_exist_on_resource: + resource_attributes.append(field.name) + return resource_attributes + + +def get_segments(resource_schema, resource): + resource_segments = [] + + if resource["category"] != "RESOURCE": + # Attributes, segments, and metrics do not have attributes + return resource_segments + + segments = resource["segments"] + for segment in segments: + if segment.startswith("segments."): + resource_segments.append(segment) + else: + segment_schema = resource_schema[segment] + segment_attributes = [ + attribute + for attribute in segment_schema["attributes"] + if attribute.startswith(f"{segment}.") + ] + resource_segments.extend(segment_attributes) + return resource_segments + + +def build_resource_metadata(api_objects, resource): + attributes = get_attributes(api_objects, resource) + + # These are the data types returned from google. They are mapped to json schema. UNSPECIFIED and UNKNOWN have never been returned. + # 0: "UNSPECIFIED", 1: "UNKNOWN", 2: "BOOLEAN", 3: "DATE", 4: "DOUBLE", 5: "ENUM", 6: "FLOAT", 7: "INT32", 8: "INT64", 9: "MESSAGE", 10: "RESOURCE_NAME", 11: "STRING", 12: "UINT64" + data_type_map = { + 0: {"type": ["null", "string"]}, + 1: {"type": ["null", "string"]}, + 2: {"type": ["null", "boolean"]}, + 3: {"type": ["null", "string"], "format": "date-time"}, + 4: {"type": ["null", "string"], "format": "singer.decimal"}, + 5: {"type": ["null", "string"]}, + 6: {"type": ["null", "string"], "format": "singer.decimal"}, + 7: {"type": ["null", "integer"]}, + 8: {"type": ["null", "integer"]}, + 9: {"type": ["null", "object", "string"], "properties": {}}, + 10: {"type": ["null", "object", "string"], "properties": {}}, + 11: {"type": ["null", "string"]}, + 12: {"type": ["null", "integer"]}, + } + + resource_metadata = { + "name": resource.name, + "category": CATEGORY_MAP[resource.category], + "json_schema": dict(data_type_map[resource.data_type]), + "selectable": resource.selectable, + "filterable": resource.filterable, + "sortable": resource.sortable, + "selectable_with": set(resource.selectable_with), + "metrics": list(resource.metrics), + "segments": list(resource.segments), + "attributes": attributes, + } + + return resource_metadata + + +def get_root_resource_name(field_name): + if not (field_name.startswith("segments.") or field_name.startswith("metrics.")): + field_root_resource = field_name.split(".")[0] + else: + field_root_resource = field_name + + return field_root_resource + + +def create_resource_schema(config): + """ + The resource schema is necessary to create a 'source of truth' with regards to the fields + Google Ads can return to us. It allows for the discovery of field exclusions and other fun + things like data types. + + + It includes every field Google Ads can return and the possible fields that each resource + can return. + + This schema is based off of the Google Ads blog posts for the creation of their query builder: + https://ads-developers.googleblog.com/2021/04/the-query-builder-blog-series-part-3.html + """ + + resource_schema = {} + + api_objects = get_api_objects(config) + + for resource in api_objects: + resource_schema[resource.name] = build_resource_metadata(api_objects, resource) + + for resource in resource_schema.values(): + updated_segments = get_segments(resource_schema, resource) + resource["segments"] = updated_segments + + for report in REPORTS: + report_object = resource_schema[report] + fields = {} + attributes = report_object["attributes"] + metrics = report_object["metrics"] + segments = report_object["segments"] + for field in attributes + metrics + segments: + field_schema = dict(resource_schema[field]) + + fields[field_schema["name"]] = { + "field_details": field_schema, + "incompatible_fields": [], + } + + # Start discovery of field exclusions + metrics_and_segments = set(metrics + segments) + + for field_name, field in fields.items(): + if field["field_details"]["category"] == "ATTRIBUTE": + continue + for compared_field in metrics_and_segments: + field_root_resource = get_root_resource_name(field_name) + compared_field_root_resource = get_root_resource_name(compared_field) + + # Fields can be any of the categories in CATEGORY_MAP, but only METRIC & SEGMENT have exclusions, so only check those + if ( + field_name != compared_field + and not compared_field.startswith(f"{field_root_resource}.") + ) and ( + fields[compared_field]["field_details"]["category"] == "METRIC" + or fields[compared_field]["field_details"]["category"] == "SEGMENT" + ): + + field_to_check = field_root_resource or field_name + compared_field_to_check = compared_field_root_resource or compared_field + + # Metrics will not be incompatible with other metrics, so don't check those + if field_name.startswith("metrics.") and compared_field.startswith("metrics."): + continue + + # If a resource is selectable with another resource they should be in + # each other's 'selectable_with' list, but Google is missing some of + # these so we have to check both ways + if ( + field_to_check not in resource_schema[compared_field_to_check]["selectable_with"] + and compared_field_to_check not in resource_schema[field_to_check]["selectable_with"] + ): + field["incompatible_fields"].append(compared_field) + + report_object["fields"] = fields + return resource_schema + + +def do_discover_streams(stream_name_to_resource): + + streams = [] + for stream_name, stream in stream_name_to_resource.items(): + + catalog_entry = { + "tap_stream_id": stream_name, + "stream": stream_name, + "schema": stream.stream_schema, + "metadata": singer.metadata.to_list(stream.stream_metadata), + } + streams.append(catalog_entry) + + return streams + + +def do_discover(resource_schema): + core_streams = do_discover_streams(initialize_core_streams(resource_schema)) + report_streams = do_discover_streams(initialize_reports(resource_schema)) + streams = [] + streams.extend(core_streams) + streams.extend(report_streams) + json.dump({"streams": streams}, sys.stdout, indent=2) diff --git a/tap_google_ads/report_definitions.py b/tap_google_ads/report_definitions.py index c940c39..071ffa7 100644 --- a/tap_google_ads/report_definitions.py +++ b/tap_google_ads/report_definitions.py @@ -1,119 +1,1839 @@ ACCOUNT_FIELDS = [] -AD_GROUP_FIELDS = [] +AD_GROUP_FIELDS = [] AD_GROUP_AD_FIELDS = [] CAMPAIGN_FIELDS = [] BIDDING_STRATEGY_FIELDS = [] ACCESSIBLE_BIDDING_STRATEGY_FIELDS = [] CAMPAIGN_BUDGET_FIELDS = [] -ACCOUNT_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'customer.manager', 'segments.click_type', 'metrics.clicks', 'metrics.content_budget_lost_impression_share', 'metrics.content_impression_share', 'metrics.content_rank_lost_impression_share', 'segments.conversion_adjustment', 'segments.conversion_or_adjustment_lag_bucket', 'segments.conversion_action_category', 'segments.conversion_lag_bucket', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'segments.hour', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'metrics.invalid_click_rate', 'metrics.invalid_clicks', 'customer.auto_tagging_enabled', 'customer.test_account', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'metrics.search_budget_lost_impression_share', 'metrics.search_exact_match_impression_share', 'metrics.search_impression_share', 'metrics.search_rank_lost_impression_share', 'segments.slot', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -ADGROUP_PERFORMANCE_REPORT_FIELDS = ['metrics.absolute_top_impression_percentage', 'customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'ad_group.type', 'segments.ad_network_type', 'ad_group.ad_rotation_mode', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'metrics.average_page_views', 'metrics.average_time_on_site', 'ad_group.base_ad_group', 'campaign.base_campaign', 'campaign.bidding_strategy', 'campaign.bidding_strategy_type', 'metrics.bounce_rate', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'ad_group.display_custom_bid_dimension', 'metrics.content_impression_share', 'metrics.content_rank_lost_impression_share', 'segments.conversion_adjustment', 'segments.conversion_or_adjustment_lag_bucket', 'segments.conversion_action_category', 'segments.conversion_lag_bucket', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cost_per_current_model_attributed_conversion', 'ad_group.cpc_bid_micros', 'ad_group.cpm_bid_micros', 'ad_group.cpv_bid_micros', 'metrics.cross_device_conversions', 'metrics.ctr', 'metrics.current_model_attributed_conversions_value', 'metrics.current_model_attributed_conversions', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'ad_group.effective_target_roas', 'ad_group.effective_target_roas_source', 'metrics.engagement_rate', 'metrics.engagements', 'campaign.manual_cpc.enhanced_cpc_enabled', 'campaign.percent_cpc.enhanced_cpc_enabled', 'segments.external_conversion_source', 'customer.id', 'ad_group.final_url_suffix', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'segments.hour', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'label.resource_name', 'segments.month', 'segments.month_of_year', 'metrics.phone_impressions', 'metrics.phone_calls', 'metrics.phone_through_rate', 'metrics.percent_new_visitors', 'segments.quarter', 'metrics.relative_ctr', 'metrics.search_absolute_top_impression_share', 'metrics.search_budget_lost_absolute_top_impression_share', 'metrics.search_budget_lost_top_impression_share', 'metrics.search_exact_match_impression_share', 'metrics.search_impression_share', 'metrics.search_rank_lost_absolute_top_impression_share', 'metrics.search_rank_lost_impression_share', 'metrics.search_rank_lost_top_impression_share', 'metrics.search_top_impression_share', 'segments.slot', 'ad_group.effective_target_cpa_micros', 'ad_group.effective_target_cpa_source', 'metrics.top_impression_percentage', 'ad_group.tracking_url_template', 'ad_group.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.value_per_current_model_attributed_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -AD_PERFORMANCE_REPORT_FIELDS = ['metrics.absolute_top_impression_percentage', 'ad_group_ad.ad.legacy_responsive_display_ad.accent_color', 'customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'ad_group_ad.ad_strength', 'ad_group_ad.ad.type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color', 'ad_group_ad.ad.added_by_google_ads', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'metrics.average_page_views', 'metrics.average_time_on_site', 'ad_group.base_ad_group', 'campaign.base_campaign', 'metrics.bounce_rate', 'ad_group_ad.ad.legacy_responsive_display_ad.business_name', 'ad_group_ad.ad.call_ad.phone_number', 'ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'ad_group_ad.policy_summary.approval_status', 'segments.conversion_adjustment', 'segments.conversion_or_adjustment_lag_bucket', 'segments.conversion_action_category', 'segments.conversion_lag_bucket', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cost_per_current_model_attributed_conversion', 'ad_group_ad.ad.final_mobile_urls', 'ad_group_ad.ad.final_urls', 'ad_group_ad.ad.tracking_url_template', 'ad_group_ad.ad.url_custom_parameters', 'segments.keyword.ad_group_criterion', 'metrics.cross_device_conversions', 'metrics.ctr', 'metrics.current_model_attributed_conversions_value', 'metrics.current_model_attributed_conversions', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', - 'ad_group_ad.ad.legacy_responsive_display_ad.description', 'ad_group_ad.ad.expanded_text_ad.description', - 'ad_group_ad.ad.text_ad.description1', 'ad_group_ad.ad.call_ad.description1', - 'ad_group_ad.ad.text_ad.description2', 'ad_group_ad.ad.call_ad.description2', - 'ad_group_criterion.negative', - 'segments.device', 'ad_group_ad.ad.device_preference', 'ad_group_ad.ad.display_url', 'metrics.engagement_rate', 'metrics.engagements', 'ad_group_ad.ad.legacy_responsive_display_ad.logo_image', 'ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image', 'ad_group_ad.ad.legacy_responsive_display_ad.marketing_image', 'ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image', 'ad_group_ad.ad.expanded_dynamic_search_ad.description', 'ad_group_ad.ad.expanded_text_ad.description2', 'ad_group_ad.ad.expanded_text_ad.headline_part3', 'segments.external_conversion_source', 'customer.id', 'ad_group_ad.ad.legacy_responsive_display_ad.format_setting', 'ad_group_ad.ad.gmail_ad.header_image', 'ad_group_ad.ad.gmail_ad.teaser.logo_image', 'ad_group_ad.ad.gmail_ad.marketing_image', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'ad_group_ad.ad.gmail_ad.teaser.business_name', 'ad_group_ad.ad.gmail_ad.teaser.description', 'ad_group_ad.ad.gmail_ad.teaser.headline', 'ad_group_ad.ad.text_ad.headline', 'ad_group_ad.ad.expanded_text_ad.headline_part1', 'ad_group_ad.ad.expanded_text_ad.headline_part2', 'ad_group_ad.ad.id', 'ad_group_ad.ad.image_ad.image_url', 'ad_group_ad.ad.image_ad.pixel_height', 'ad_group_ad.ad.image_ad.pixel_width', 'ad_group_ad.ad.image_ad.mime_type', 'ad_group_ad.ad.image_ad.name', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'label.resource_name', 'label.name', 'ad_group_ad.ad.legacy_responsive_display_ad.long_headline', 'ad_group_ad.ad.legacy_responsive_display_ad.main_color', 'ad_group_ad.ad.gmail_ad.marketing_image_display_call_to_action.text', 'ad_group_ad.ad.gmail_ad.marketing_image_display_call_to_action.text_color', 'ad_group_ad.ad.gmail_ad.marketing_image_headline', 'ad_group_ad.ad.gmail_ad.marketing_image_description', 'segments.month', 'segments.month_of_year', 'ad_group_ad.ad.responsive_display_ad.accent_color', 'ad_group_ad.ad.responsive_display_ad.allow_flexible_color', 'ad_group_ad.ad.responsive_display_ad.business_name', 'ad_group_ad.ad.responsive_display_ad.call_to_action_text', 'ad_group_ad.ad.responsive_display_ad.descriptions', 'ad_group_ad.ad.responsive_display_ad.price_prefix', 'ad_group_ad.ad.responsive_display_ad.promo_text', 'ad_group_ad.ad.responsive_display_ad.format_setting', 'ad_group_ad.ad.responsive_display_ad.headlines', 'ad_group_ad.ad.responsive_display_ad.logo_images', 'ad_group_ad.ad.responsive_display_ad.square_logo_images', 'ad_group_ad.ad.responsive_display_ad.long_headline', 'ad_group_ad.ad.responsive_display_ad.main_color', 'ad_group_ad.ad.responsive_display_ad.marketing_images', 'ad_group_ad.ad.responsive_display_ad.square_marketing_images', 'ad_group_ad.ad.responsive_display_ad.youtube_videos', 'ad_group_ad.ad.expanded_text_ad.path1', 'ad_group_ad.ad.expanded_text_ad.path2', 'metrics.percent_new_visitors', - 'ad_group_ad.policy_summary.policy_topic_entries', - #'ad_group_ad.policy_summary.review_state', - 'ad_group_ad.policy_summary.review_status', - 'ad_group_ad.policy_summary.approval_status', - 'ad_group_ad.ad.legacy_responsive_display_ad.price_prefix', 'ad_group_ad.ad.legacy_responsive_display_ad.promo_text', 'segments.quarter', 'ad_group_ad.ad.responsive_search_ad.descriptions', 'ad_group_ad.ad.responsive_search_ad.headlines', 'ad_group_ad.ad.responsive_search_ad.path1', 'ad_group_ad.ad.responsive_search_ad.path2', 'ad_group_ad.ad.legacy_responsive_display_ad.short_headline', 'segments.slot', 'ad_group_ad.status', 'ad_group_ad.ad.system_managed_resource_source', 'metrics.top_impression_percentage', 'ad_group_ad.ad.app_ad.descriptions', 'ad_group_ad.ad.app_ad.headlines', 'ad_group_ad.ad.app_ad.html5_media_bundles', 'ad_group_ad.ad.app_ad.images', 'ad_group_ad.ad.app_ad.mandatory_ad_text', 'ad_group_ad.ad.app_ad.youtube_videos', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.value_per_current_model_attributed_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -AGE_RANGE_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'ad_group.base_ad_group', 'campaign.base_campaign', 'ad_group_criterion.bid_modifier', 'campaign.bidding_strategy', 'bidding_strategy.name', 'campaign.bidding_strategy_type', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'ad_group_criterion.effective_cpc_bid_micros', 'ad_group_criterion.effective_cpc_bid_source', 'ad_group_criterion.effective_cpm_bid_micros', 'ad_group_criterion.effective_cpm_bid_source', 'ad_group_criterion.age_range.type', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'ad_group_criterion.final_mobile_urls', 'ad_group_criterion.final_urls', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'ad_group_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'ad_group_criterion.negative', 'ad_group.targeting_setting.target_restrictions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'ad_group_criterion.status', 'ad_group_criterion.tracking_url_template', 'ad_group_criterion.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -AUDIENCE_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'ad_group.base_ad_group', 'campaign.base_campaign', 'campaign.bidding_strategy', - 'campaign_criterion.bid_modifier', - 'ad_group_criterion.bid_modifier', - 'bidding_strategy.name', - #'campaign.bidding_strategy.type', - 'campaign.bidding_strategy_type', - 'ad_group.campaign', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', - #'This should be campaign/ad group criterion depending on the view.', - 'ad_group_criterion.effective_cpc_bid_micros', - 'ad_group_criterion.effective_cpc_bid_source', - 'ad_group_criterion.effective_cpm_bid_micros', - 'ad_group_criterion.effective_cpm_bid_source', - #'This should be campaign/ad group bid modifier depending on the view.', - 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'ad_group_criterion.final_mobile_urls', 'ad_group_criterion.final_urls', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'ad_group_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'ad_group.targeting_setting.target_restrictions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'segments.slot', 'ad_group_criterion.status', 'ad_group.tracking_url_template', 'ad_group.url_custom_parameters', 'user_list.name', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'campaign.id', 'campaign.name', 'campaign.status', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'Use group_placement_view.placement_type or group_placement_view.target_url.', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'Returns domain name for websites and YouTube channel name for YouTube channels', 'group_placement_view.target_url', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -BID_GOAL_PERFORMANCE_REPORT_FIELDS = ['customer.descriptive_name', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cpc', 'metrics.average_cpm', 'bidding_strategy.campaign_count', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'segments.date', 'segments.day_of_week', 'segments.device', 'segments.external_conversion_source', 'customer.id', 'segments.hour', 'bidding_strategy.id', 'metrics.impressions', 'segments.month', 'segments.month_of_year', 'bidding_strategy.name must be selected withy the resources bidding_strategy or campaign.', 'bidding_strategy.non_removed_campaign_count', 'segments.quarter', 'bidding_strategy.status', 'bidding_strategy.target_cpa.target_cpa_micros', 'bidding_strategy.target_cpa.cpc_bid_ceiling_micros', 'bidding_strategy.target_cpa.cpc_bid_floor_micros', 'bidding_strategy.target_roas.target_roas', 'bidding_strategy.target_roas.cpc_bid_ceiling_micros', 'bidding_strategy.target_roas.cpc_bid_floor_micros', 'bidding_strategy.target_spend.cpc_bid_ceiling_micros', 'bidding_strategy.target_spend.target_spend_micros', 'bidding_strategy.type', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -BUDGET_PERFORMANCE_REPORT_FIELDS = ['customer.descriptive_name', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'campaign_budget.amount_micros', 'campaign.id', 'campaign.name', 'campaign.status', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'segments.budget_campaign_association_status.status', 'campaign_budget.id', 'campaign_budget.name', 'campaign_budget.reference_count', 'campaign_budget.status', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'campaign_budget.delivery_method', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'campaign_budget.has_recommended_budget', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'campaign_budget.explicitly_shared', 'campaign_budget.period', 'campaign_budget.recommended_budget_amount_micros', 'campaign_budget.recommended_budget_estimated_change_weekly_clicks', 'campaign_budget.recommended_budget_estimated_change_weekly_cost_micros', 'campaign_budget.recommended_budget_estimated_change_weekly_interactions', 'campaign_budget.recommended_budget_estimated_change_weekly_views', 'campaign_budget.total_amount_micros', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions'] -CALL_METRICS_CALL_DETAILS_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'call_view.call_duration_seconds', 'call_view.end_call_date_time', 'call_view.start_call_date_time', 'call_view.call_status', 'call_view.call_tracking_display_location', 'call_view.type', 'call_view.caller_area_code', 'call_view.caller_country_code', 'campaign.id', 'campaign.name', 'campaign.status', 'customer.descriptive_name', - #'segments.date', - #'segments.day_of_week', - 'customer.id', - #'segments.hour', - #'segments.month', - #'segments.month_of_year', - #'segments.quarter', - #'segments.week', - #'segments.year' -] -CAMPAIGN_AD_SCHEDULE_TARGET_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'campaign_criterion.bid_modifier', 'campaign.id', 'campaign.name', 'campaign.status', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'campaign_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -CAMPAIGN_CRITERIA_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'campaign.base_campaign', 'campaign.id', 'campaign.name', 'campaign.status', 'campaign_criterion.keyword.text OR campaign_criterion.placement.url, etc.', 'campaign_criterion.type', 'customer.descriptive_name', 'customer.id', 'campaign_criterion.criterion_id', 'campaign_criterion.negative'] -CAMPAIGN_LOCATION_TARGET_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'campaign_criterion.bid_modifier', 'campaign.id', 'campaign.name', 'campaign.status', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'campaign_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'campaign_criterion.negative', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -CAMPAIGN_PERFORMANCE_REPORT_FIELDS = ['metrics.absolute_top_impression_percentage', 'customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'segments.ad_network_type', 'campaign.advertising_channel_sub_type', 'campaign.advertising_channel_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'campaign_budget.amount_micros', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'metrics.average_page_views', 'metrics.average_time_on_site', 'campaign.base_campaign', 'campaign.bidding_strategy', 'bidding_strategy.name', 'campaign.bidding_strategy_type', 'metrics.bounce_rate', 'campaign.campaign_budget', - # "Not available with 'FROM campaign'. However, you can retrieve device type bid modifiers via 'FROM campaign_criterion' queries.", - 'campaign_criterion.device.type', - 'campaign.id', - #"Not available with 'FROM campaign'. However, you can retrieve device type bid modifiers via 'FROM campaign_criterion' queries.", - 'campaign.name', 'campaign.status', - #"Not available with 'FROM campaign'. However, you can retrieve device type bid modifiers via 'FROM campaign_criterion' queries.", - 'campaign.experiment_type', 'segments.click_type', 'metrics.clicks', 'metrics.content_budget_lost_impression_share', 'metrics.content_impression_share', 'metrics.content_rank_lost_impression_share', 'segments.conversion_adjustment', 'segments.conversion_or_adjustment_lag_bucket', 'segments.conversion_attribution_event_type', 'segments.conversion_action_category', 'segments.conversion_lag_bucket', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cost_per_current_model_attributed_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'metrics.current_model_attributed_conversions_value', 'metrics.current_model_attributed_conversions', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'campaign.end_date', 'metrics.engagement_rate', 'metrics.engagements', 'campaign.manual_cpc.enhanced_cpc_enabled', 'campaign.percent_cpc.enhanced_cpc_enabled', 'segments.external_conversion_source', 'customer.id', 'campaign.final_url_suffix', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'campaign_budget.has_recommended_budget', 'segments.hour', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'metrics.invalid_click_rate', 'metrics.invalid_clicks', 'campaign_budget.explicitly_shared', - #'Select label.resource_name from the resource campaign_label', - #'Select label.resource_name from the resource campaign_label', - 'label.resource_name', - 'campaign.maximize_conversion_value.target_roas', 'segments.month', 'segments.month_of_year', 'metrics.phone_impressions', 'metrics.phone_calls', 'metrics.phone_through_rate', 'metrics.percent_new_visitors', 'campaign_budget.period', 'segments.quarter', 'campaign_budget.recommended_budget_amount_micros', 'metrics.relative_ctr', 'metrics.search_absolute_top_impression_share', 'metrics.search_budget_lost_absolute_top_impression_share', 'metrics.search_budget_lost_impression_share', 'metrics.search_budget_lost_top_impression_share', 'metrics.search_click_share', 'metrics.search_exact_match_impression_share', 'metrics.search_impression_share', 'metrics.search_rank_lost_absolute_top_impression_share', 'metrics.search_rank_lost_impression_share', 'metrics.search_rank_lost_top_impression_share', 'metrics.search_top_impression_share', 'campaign.serving_status', 'segments.slot', 'campaign.start_date', 'metrics.top_impression_percentage', 'campaign_budget.total_amount_micros', 'campaign.tracking_url_template', 'campaign.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.value_per_current_model_attributed_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -CAMPAIGN_SHARED_SET_REPORT_FIELDS = ['customer.descriptive_name', 'campaign.id', 'campaign.name', 'campaign.status', 'customer.id', 'shared_set.id', 'shared_set.name', 'shared_set.type', 'campaign_shared_set.status'] -CLICK_PERFORMANCE_REPORT_FIELDS = ['customer.descriptive_name', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'click_view.area_of_interest.city', 'click_view.area_of_interest.country', 'click_view.area_of_interest.metro', 'click_view.area_of_interest.most_specific', 'click_view.area_of_interest.region', 'campaign.id', 'click_view.campaign_location_target', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'click_view.ad_group_ad', 'segments.date', 'segments.device', 'customer.id', 'click_view.gclid', 'click_view.location_of_presence.city', 'click_view.location_of_presence.country', 'click_view.location_of_presence.metro', 'click_view.location_of_presence.most_specific', 'click_view.location_of_presence.region', 'segments.month_of_year', 'click_view.page_number', 'segments.slot', 'click_view.user_list'] -DISPLAY_KEYWORD_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'ad_group.base_ad_group', 'campaign.base_campaign', 'campaign.bidding_strategy', - #'bidding_strategy.name must be selected with the resources bidding_strategy and campaign.', - 'bidding_strategy.name', - 'campaign.bidding_strategy_type', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'ad_group_criterion.effective_cpc_bid_micros', 'ad_group_criterion.effective_cpc_bid_source', 'ad_group_criterion.effective_cpm_bid_micros', 'ad_group_criterion.effective_cpm_bid_source', 'ad_group_criterion.effective_cpv_bid_micros', 'ad_group_criterion.effective_cpv_bid_source', 'ad_group_criterion.keyword.text', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'ad_group_criterion.final_mobile_urls', 'ad_group_criterion.final_urls', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'ad_group_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'ad_group_criterion.negative', 'ad_group.targeting_setting.target_restrictions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'ad_group_criterion.status', 'ad_group_criterion.tracking_url_template', 'ad_group_criterion.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -DISPLAY_TOPICS_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'ad_group.base_ad_group', 'campaign.base_campaign', 'ad_group_criterion.bid_modifier', 'campaign.bidding_strategy', - #'bidding_strategy.name must be selected with the resources bidding_strategy and campaign.', - 'bidding_strategy.name', - 'campaign.bidding_strategy_type', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'ad_group_criterion.effective_cpc_bid_micros', 'ad_group_criterion.effective_cpc_bid_source', 'ad_group_criterion.effective_cpm_bid_micros', 'ad_group_criterion.effective_cpm_bid_source', 'ad_group_criterion.topic.path', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'ad_group_criterion.final_mobile_urls', 'ad_group_criterion.final_urls', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'ad_group_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'ad_group_criterion.negative', 'ad_group.targeting_setting.target_restrictions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'ad_group_criterion.status', 'ad_group_criterion.tracking_url_template', 'ad_group_criterion.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'ad_group_criterion.topic.topic_constant', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -GENDER_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'ad_group.base_ad_group', 'campaign.base_campaign', 'ad_group_criterion.bid_modifier', 'campaign.bidding_strategy', 'bidding_strategy.name', 'bidding_strategy.type', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'ad_group_criterion.effective_cpc_bid_micros', 'ad_group_criterion.effective_cpc_bid_source', 'ad_group_criterion.effective_cpm_bid_micros', 'ad_group_criterion.effective_cpm_bid_source', 'ad_group_criterion.gender.type', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'ad_group_criterion.final_mobile_urls', 'ad_group_criterion.final_urls', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'ad_group_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'ad_group_criterion.negative', 'ad_group.targeting_setting.target_restrictions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'ad_group_criterion.status', 'ad_group_criterion.tracking_url_template', 'ad_group_criterion.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -GEO_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpm', 'metrics.average_cpv', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.geo_target_city', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', - #'Use geographic_view.country_criterion_id or user_location_view.country_criterion_id depending upon which view you want', - 'geographic_view.country_criterion_id', - 'user_location_view.country_criterion_id', - 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'segments.external_conversion_source', 'customer.id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'user_location_view.targeting_location', 'geographic_view.location_type', 'segments.geo_target_metro', 'segments.month', 'segments.month_of_year', 'segments.geo_target_most_specific_location', 'segments.quarter', 'segments.geo_target_region', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -KEYWORDLESS_QUERY_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cpc', 'metrics.average_cpm', 'campaign.id', 'campaign.name', 'campaign.status', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'segments.webpage', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.external_conversion_source', 'customer.id', 'metrics.impressions', 'dynamic_search_ads_search_term_view.headline', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'dynamic_search_ads_search_term_view.search_term', 'dynamic_search_ads_search_term_view.landing_page', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'segments.week', 'segments.year'] -KEYWORDS_PERFORMANCE_REPORT_FIELDS = ['metrics.absolute_top_impression_percentage', 'customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'ad_group_criterion.approval_status', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'metrics.average_page_views', 'metrics.average_time_on_site', 'ad_group.base_ad_group', 'campaign.base_campaign', 'campaign.bidding_strategy', 'campaign.bidding_strategy_type', 'metrics.bounce_rate', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_adjustment', 'segments.conversion_or_adjustment_lag_bucket', 'segments.conversion_action_category', 'segments.conversion_lag_bucket', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cost_per_current_model_attributed_conversion', 'ad_group_criterion.effective_cpc_bid_micros', 'ad_group_criterion.effective_cpc_bid_source', 'ad_group_criterion.effective_cpm_bid_micros', 'ad_group_criterion.quality_info.creative_quality_score', 'ad_group_criterion.keyword.text', 'metrics.cross_device_conversions', 'metrics.ctr', 'metrics.current_model_attributed_conversions_value', 'metrics.current_model_attributed_conversions', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', - # 'campaign.manual_cpc.enhanced_cpc_enabled || campaign.percent_cpc.enhanced_cpc_enabled', - 'campaign.manual_cpc.enhanced_cpc_enabled', - 'campaign.percent_cpc.enhanced_cpc_enabled', - 'ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc', 'ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc', 'segments.external_conversion_source', 'customer.id', 'ad_group_criterion.final_mobile_urls', 'ad_group_criterion.final_url_suffix', 'ad_group_criterion.final_urls', 'ad_group_criterion.position_estimates.first_page_cpc_micros', 'ad_group_criterion.position_estimates.first_position_cpc_micros', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'metrics.historical_creative_quality_score', 'metrics.historical_landing_page_quality_score', 'metrics.historical_quality_score', 'metrics.historical_search_predicted_ctr', 'ad_group_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'ad_group_criterion.negative', 'ad_group_criterion.keyword.match_type', - #'Select label.resource_name from the resource ad_group_label', - #'Select label.name from the resource ad_group_label', - 'label.resource_name', - 'label.name', - 'segments.month', 'segments.month_of_year', 'metrics.percent_new_visitors', 'ad_group_criterion.quality_info.post_click_quality_score', 'ad_group_criterion.quality_info.quality_score', 'segments.quarter', 'metrics.search_absolute_top_impression_share', 'metrics.search_budget_lost_absolute_top_impression_share', 'metrics.search_budget_lost_top_impression_share', 'metrics.search_exact_match_impression_share', 'metrics.search_impression_share', 'ad_group_criterion.quality_info.search_predicted_ctr', 'metrics.search_rank_lost_absolute_top_impression_share', 'metrics.search_rank_lost_impression_share', 'metrics.search_rank_lost_top_impression_share', 'metrics.search_top_impression_share', 'segments.slot', 'ad_group_criterion.status', 'ad_group_criterion.system_serving_status', 'metrics.top_impression_percentage', 'ad_group_criterion.position_estimates.top_of_page_cpc_micros', 'ad_group_criterion.tracking_url_template', 'ad_group_criterion.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.value_per_current_model_attributed_conversion', 'ad_group_criterion.topic.topic_constant', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -LABEL_REPORT_FIELDS = ['customer.descriptive_name', 'customer.id', 'label.id', 'label.name', 'deprecated'] -LANDING_PAGE_REPORT_FIELDS = ['metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'campaign.advertising_channel_type', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'metrics.conversions_from_interactions_rate', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'expanded_landing_page_view.expanded_final_url', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'segments.month', 'segments.month_of_year', 'metrics.mobile_friendly_clicks_percentage', 'metrics.valid_accelerated_mobile_pages_clicks_percentage', 'segments.quarter', 'segments.slot', 'metrics.speed_score', 'landing_page_view.unexpanded_final_url', 'metrics.value_per_conversion', 'metrics.video_view_rate', 'metrics.video_views', 'segments.week', 'segments.year'] -PAID_ORGANIC_QUERY_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'metrics.average_cpc', 'campaign.id', 'campaign.name', 'campaign.status', 'metrics.clicks', 'metrics.combined_clicks', 'metrics.combined_clicks_per_query', 'metrics.combined_queries', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'customer.id', 'metrics.impressions', 'segments.keyword.ad_group_criterion', 'segments.keyword.info.text', 'segments.month', 'segments.month_of_year', 'metrics.organic_clicks', 'metrics.organic_clicks_per_query', 'metrics.organic_impressions', 'metrics.organic_impressions_per_query', 'metrics.organic_queries', 'segments.quarter', 'paid_organic_search_term_view.search_term', 'segments.search_engine_results_page_type', 'segments.week', 'segments.year'] -PARENTAL_STATUS_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'ad_group.base_ad_group', 'campaign.base_campaign', 'campaign.bidding_strategy', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'ad_group_criterion.effective_cpc_bid_micros', 'ad_group_criterion.effective_cpc_bid_source', 'ad_group_criterion.effective_cpm_bid_micros', 'ad_group_criterion.effective_cpm_bid_source', 'ad_group_criterion.parental_status.type', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'ad_group_criterion.final_mobile_urls', 'ad_group_criterion.final_urls', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'ad_group_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'ad_group_criterion.negative', 'ad_group.targeting_setting.target_restrictions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'ad_group_criterion.status', 'ad_group_criterion.tracking_url_template', 'ad_group_criterion.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -PLACEHOLDER_FEED_ITEM_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'ad_group_ad.resource_name', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'feed_item.attribute_values', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', - #'feed_item_target.device is available with FROM feed_item_target', - 'feed_item_target.device', - #'See feed_item.policy_infos for policy information.', - 'feed_item.policy_infos', - 'feed_item.end_date_time', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'feed_item.feed', 'feed_item.id', 'feed_item_target.feed_item_target_id', 'feed_item.geo_targeting_restriction', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'segments.interaction_on_this_extension', 'feed_item_target.keyword.match_type', 'feed_item_target.feed_item_target_id', 'feed_item_target.keyword.match_type', 'feed_item_target.keyword.text', 'segments.month', 'segments.month_of_year', 'segments.placeholder_type', 'segments.quarter', 'feed_item_target.ad_schedule', 'segments.slot', 'feed_item.start_date_time', 'feed_item.status', 'feed_item_target.ad_group', 'feed_item_target.campaign', 'feed_item.url_custom_parameters', - #'See feed_item.policy_infos for policy information.', - 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_view_rate', 'metrics.video_views', 'segments.week', 'segments.year'] -PLACEHOLDER_REPORT_FIELDS = ['customer.descriptive_name', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', - #'campaign', - 'campaign.id', - 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'ad_group_ad.resource_name', 'feed_placeholder_view.placeholder_type', 'segments.external_conversion_source', 'customer.id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'segments.slot', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -PLACEMENT_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'ad_group.base_ad_group', 'campaign.base_campaign', 'ad_group_criterion.bid_modifier', 'campaign.bidding_strategy', 'bidding_strategy.name', 'bidding_strategy.type', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.all_conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'ad_group_criterion.effective_cpc_bid_micros', 'ad_group_criterion.effective_cpc_bid_source', 'ad_group_criterion.effective_cpm_bid_micros', 'ad_group_criterion.effective_cpm_bid_source', 'ad_group_criterion.placement.url', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', - #'Returns browser url for websites and for YouTube, video and channel.', - # BUG We don't know what to do with this. We looked for a browser url type thing in the query builder - 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'ad_group_criterion.final_mobile_urls', 'ad_group_criterion.final_urls', 'metrics.gmail_forwards', 'metrics.gmail_saves', 'metrics.gmail_secondary_clicks', 'ad_group_criterion.criterion_id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'ad_group_criterion.negative', 'ad_group.targeting_setting.target_restrictions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'ad_group_criterion.status', 'ad_group_criterion.tracking_url_template', 'ad_group_criterion.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -PRODUCT_PARTITION_REPORT_FIELDS = ['customer.descriptive_name', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cpc', 'metrics.average_cpm', 'metrics.benchmark_average_max_cpc', 'metrics.benchmark_ctr', 'campaign.bidding_strategy_type', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'ad_group_criterion.cpc_bid_micros', 'metrics.cross_device_conversions', 'metrics.ctr', 'segments.date', 'segments.day_of_week', 'segments.device', 'segments.external_conversion_source', 'customer.id', 'ad_group_criterion.final_url_suffix', 'ad_group_criterion.criterion_id', 'metrics.impressions', 'ad_group_criterion.negative', 'segments.month', 'segments.month_of_year', 'ad_group_criterion.listing_group.parent_ad_group_criterion', 'ad_group_criterion.listing_group.type', 'segments.quarter', 'metrics.search_absolute_top_impression_share', 'metrics.search_click_share', 'metrics.search_impression_share', 'ad_group_criterion.tracking_url_template', 'ad_group_criterion.url_custom_parameters', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -# RESOURCES = ['customer', 'ad_group_ad', 'ad_group', 'age_range_view', 'campaign_audience_view', 'group_placement_view', 'bidding_strategy', 'campaign_budget', 'call_view', 'ad_schedule_view', 'campaign_criterion', 'campaign', 'campaign_shared_set', 'location_view', 'click_view', 'display_keyword_view', 'topic_view', 'gender_view', 'geographic_view', 'dynamic_search_ads_search_term_view', 'keyword_view', 'label', 'landing_page_view', 'paid_organic_search_term_view', 'parental_status_view', 'feed_item', 'feed_placeholder_view', 'managed_placement_view', 'product_group_view', 'search_term_view', 'shared_criterion', 'shared_set', 'shopping_performance_view', 'detail_placement_view', 'distance_view', 'video'] -SEARCH_QUERY_PERFORMANCE_REPORT_FIELDS = ['metrics.absolute_top_impression_percentage', 'customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'campaign.id', 'campaign.name', 'campaign.status', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'ad_group_ad.ad.id', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'ad_group_ad.ad.final_urls', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'segments.keyword.ad_group_criterion', 'segments.keyword.info.text', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'search_term_view.search_term', 'segments.search_term_match_type', 'search_term_view.status', 'metrics.top_impression_percentage', 'ad_group_ad.ad.tracking_url_template', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -SHARED_SET_CRITERIA_REPORT_FIELDS = ['customer.descriptive_name', 'shared_criterion.keyword.text OR shared_criterion.placement.url, etc.', 'customer.id', 'shared_criterion.criterion_id', 'shared_criterion.keyword.match_type', 'shared_set.id'] -SHOPPING_PERFORMANCE_REPORT_FIELDS = ['customer.descriptive_name', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'segments.product_aggregator_id', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cpc', 'segments.product_brand', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.product_bidding_category_level1', 'segments.product_bidding_category_level2', 'segments.product_bidding_category_level3', 'segments.product_bidding_category_level4', 'segments.product_bidding_category_level5', 'segments.product_channel', 'segments.product_channel_exclusivity', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'segments.product_country', 'metrics.cross_device_conversions', 'metrics.ctr', 'segments.product_custom_attribute0', 'segments.product_custom_attribute1', 'segments.product_custom_attribute2', 'segments.product_custom_attribute3', 'segments.product_custom_attribute4', 'segments.date', 'segments.day_of_week', 'segments.device', 'segments.external_conversion_source', 'customer.id', 'metrics.impressions', 'segments.product_language', 'segments.product_merchant_id', 'segments.month', 'segments.product_item_id', 'segments.product_condition', 'segments.product_title', 'segments.product_type_l1', 'segments.product_type_l2', 'segments.product_type_l3', 'segments.product_type_l4', 'segments.product_type_l5', 'segments.quarter', 'metrics.search_absolute_top_impression_share', 'metrics.search_click_share', 'metrics.search_impression_share', 'segments.product_store_id', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'segments.week', 'segments.year'] -URL_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'metrics.active_view_cpm', 'metrics.active_view_ctr', 'metrics.active_view_impressions', 'metrics.active_view_measurability', 'metrics.active_view_measurable_cost_micros', 'metrics.active_view_measurable_impressions', 'metrics.active_view_viewability', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cost', 'metrics.average_cpc', 'metrics.average_cpe', 'metrics.average_cpm', 'metrics.average_cpv', 'campaign.id', 'campaign.name', 'campaign.status', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.all_conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'detail_placement_view.display_name', 'detail_placement_view.group_placement_target_url', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'metrics.impressions', 'metrics.interaction_rate', 'metrics.interaction_event_types', 'metrics.interactions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'detail_placement_view.target_url', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -USER_AD_DISTANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cpc', 'metrics.average_cpm', 'campaign.id', 'campaign.name', 'campaign.status', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'distance_view.distance_bucket', 'segments.external_conversion_source', 'customer.id', 'metrics.impressions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'metrics.value_per_all_conversions', 'metrics.value_per_conversion', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] -VIDEO_PERFORMANCE_REPORT_FIELDS = ['customer.currency_code', 'customer.descriptive_name', 'customer.time_zone', 'ad_group.id', 'ad_group.name', 'ad_group.status', 'segments.ad_network_type', 'metrics.all_conversions_from_interactions_rate', 'metrics.all_conversions_value', 'metrics.all_conversions', 'metrics.average_cpm', 'metrics.average_cpv', 'campaign.id', 'campaign.name', 'campaign.status', 'segments.click_type', 'metrics.clicks', 'segments.conversion_action_category', 'metrics.all_conversions_from_interactions_rate', 'segments.conversion_action', 'segments.conversion_action_name', 'metrics.conversions_value', 'metrics.conversions', 'metrics.cost_micros', 'metrics.cost_per_all_conversions', 'metrics.cost_per_conversion', 'ad_group_ad.ad.id', 'ad_group_ad.status', 'metrics.cross_device_conversions', 'metrics.ctr', 'customer.descriptive_name', 'segments.date', 'segments.day_of_week', 'segments.device', 'metrics.engagement_rate', 'metrics.engagements', 'segments.external_conversion_source', 'customer.id', 'metrics.impressions', 'segments.month', 'segments.month_of_year', 'segments.quarter', 'metrics.value_per_all_conversions', 'video.channel_id', 'video.duration_millis', 'video.id', 'metrics.video_quartile_p100_rate', 'metrics.video_quartile_p25_rate', 'metrics.video_quartile_p50_rate', 'metrics.video_quartile_p75_rate', 'video.title', 'metrics.video_view_rate', 'metrics.video_views', 'metrics.view_through_conversions', 'segments.week', 'segments.year'] +ACCOUNT_PERFORMANCE_REPORT_FIELDS = [ + "customer.auto_tagging_enabled", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.manager", + "customer.test_account", + "customer.time_zone", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.content_budget_lost_impression_share", + "metrics.content_impression_share", + "metrics.content_rank_lost_impression_share", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.invalid_click_rate", + "metrics.invalid_clicks", + "metrics.search_budget_lost_impression_share", + "metrics.search_exact_match_impression_share", + "metrics.search_impression_share", + "metrics.search_rank_lost_impression_share", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.conversion_adjustment", + "segments.conversion_lag_bucket", + "segments.conversion_or_adjustment_lag_bucket", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.hour", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", +] +ADGROUP_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.ad_rotation_mode", + "ad_group.base_ad_group", + "ad_group.cpc_bid_micros", + "ad_group.cpm_bid_micros", + "ad_group.cpv_bid_micros", + "ad_group.display_custom_bid_dimension", + "ad_group.effective_target_cpa_micros", + "ad_group.effective_target_cpa_source", + "ad_group.effective_target_roas", + "ad_group.effective_target_roas_source", + "ad_group.final_url_suffix", + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group.tracking_url_template", + "ad_group.type", + "ad_group.url_custom_parameters", + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.bidding_strategy_type", + "campaign.id", + "campaign.manual_cpc.enhanced_cpc_enabled", + "campaign.name", + "campaign.percent_cpc.enhanced_cpc_enabled", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.absolute_top_impression_percentage", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.average_page_views", + "metrics.average_time_on_site", + "metrics.bounce_rate", + "metrics.clicks", + "metrics.content_impression_share", + "metrics.content_rank_lost_impression_share", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cost_per_current_model_attributed_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.current_model_attributed_conversions", + "metrics.current_model_attributed_conversions_value", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.percent_new_visitors", + "metrics.phone_calls", + "metrics.phone_impressions", + "metrics.phone_through_rate", + "metrics.relative_ctr", + "metrics.search_absolute_top_impression_share", + "metrics.search_budget_lost_absolute_top_impression_share", + "metrics.search_budget_lost_top_impression_share", + "metrics.search_exact_match_impression_share", + "metrics.search_impression_share", + "metrics.search_rank_lost_absolute_top_impression_share", + "metrics.search_rank_lost_impression_share", + "metrics.search_rank_lost_top_impression_share", + "metrics.search_top_impression_share", + "metrics.top_impression_percentage", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.value_per_current_model_attributed_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.conversion_adjustment", + "segments.conversion_lag_bucket", + "segments.conversion_or_adjustment_lag_bucket", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.hour", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", +] +AD_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.base_ad_group", + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group_ad.ad.added_by_google_ads", + "ad_group_ad.ad.app_ad.descriptions", + "ad_group_ad.ad.app_ad.headlines", + "ad_group_ad.ad.app_ad.html5_media_bundles", + "ad_group_ad.ad.app_ad.images", + "ad_group_ad.ad.app_ad.mandatory_ad_text", + "ad_group_ad.ad.app_ad.youtube_videos", + "ad_group_ad.ad.call_ad.description1", + "ad_group_ad.ad.call_ad.description2", + "ad_group_ad.ad.call_ad.phone_number", + "ad_group_ad.ad.device_preference", + "ad_group_ad.ad.display_url", + "ad_group_ad.ad.expanded_dynamic_search_ad.description", + "ad_group_ad.ad.expanded_text_ad.description", + "ad_group_ad.ad.expanded_text_ad.description2", + "ad_group_ad.ad.expanded_text_ad.headline_part1", + "ad_group_ad.ad.expanded_text_ad.headline_part2", + "ad_group_ad.ad.expanded_text_ad.headline_part3", + "ad_group_ad.ad.expanded_text_ad.path1", + "ad_group_ad.ad.expanded_text_ad.path2", + "ad_group_ad.ad.final_mobile_urls", + "ad_group_ad.ad.final_urls", + "ad_group_ad.ad.gmail_ad.header_image", + "ad_group_ad.ad.gmail_ad.marketing_image", + "ad_group_ad.ad.gmail_ad.marketing_image_description", + "ad_group_ad.ad.gmail_ad.marketing_image_display_call_to_action.text", + "ad_group_ad.ad.gmail_ad.marketing_image_display_call_to_action.text_color", + "ad_group_ad.ad.gmail_ad.marketing_image_headline", + "ad_group_ad.ad.gmail_ad.teaser.business_name", + "ad_group_ad.ad.gmail_ad.teaser.description", + "ad_group_ad.ad.gmail_ad.teaser.headline", + "ad_group_ad.ad.gmail_ad.teaser.logo_image", + "ad_group_ad.ad.id", + "ad_group_ad.ad.image_ad.image_url", + "ad_group_ad.ad.image_ad.mime_type", + "ad_group_ad.ad.image_ad.name", + "ad_group_ad.ad.image_ad.pixel_height", + "ad_group_ad.ad.image_ad.pixel_width", + "ad_group_ad.ad.legacy_responsive_display_ad.accent_color", + "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color", + "ad_group_ad.ad.legacy_responsive_display_ad.business_name", + "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text", + "ad_group_ad.ad.legacy_responsive_display_ad.description", + "ad_group_ad.ad.legacy_responsive_display_ad.format_setting", + "ad_group_ad.ad.legacy_responsive_display_ad.logo_image", + "ad_group_ad.ad.legacy_responsive_display_ad.long_headline", + "ad_group_ad.ad.legacy_responsive_display_ad.main_color", + "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image", + "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix", + "ad_group_ad.ad.legacy_responsive_display_ad.promo_text", + "ad_group_ad.ad.legacy_responsive_display_ad.short_headline", + "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image", + "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image", + "ad_group_ad.ad.responsive_display_ad.accent_color", + "ad_group_ad.ad.responsive_display_ad.allow_flexible_color", + "ad_group_ad.ad.responsive_display_ad.business_name", + "ad_group_ad.ad.responsive_display_ad.call_to_action_text", + "ad_group_ad.ad.responsive_display_ad.descriptions", + "ad_group_ad.ad.responsive_display_ad.format_setting", + "ad_group_ad.ad.responsive_display_ad.headlines", + "ad_group_ad.ad.responsive_display_ad.logo_images", + "ad_group_ad.ad.responsive_display_ad.long_headline", + "ad_group_ad.ad.responsive_display_ad.main_color", + "ad_group_ad.ad.responsive_display_ad.marketing_images", + "ad_group_ad.ad.responsive_display_ad.price_prefix", + "ad_group_ad.ad.responsive_display_ad.promo_text", + "ad_group_ad.ad.responsive_display_ad.square_logo_images", + "ad_group_ad.ad.responsive_display_ad.square_marketing_images", + "ad_group_ad.ad.responsive_display_ad.youtube_videos", + "ad_group_ad.ad.responsive_search_ad.descriptions", + "ad_group_ad.ad.responsive_search_ad.headlines", + "ad_group_ad.ad.responsive_search_ad.path1", + "ad_group_ad.ad.responsive_search_ad.path2", + "ad_group_ad.ad.system_managed_resource_source", + "ad_group_ad.ad.text_ad.description1", + "ad_group_ad.ad.text_ad.description2", + "ad_group_ad.ad.text_ad.headline", + "ad_group_ad.ad.tracking_url_template", + "ad_group_ad.ad.type", + "ad_group_ad.ad.url_custom_parameters", + "ad_group_ad.ad_strength", + "ad_group_ad.policy_summary.approval_status", + "ad_group_ad.policy_summary.approval_status", + "ad_group_ad.policy_summary.policy_topic_entries", + "ad_group_ad.policy_summary.review_status", + "ad_group_ad.status", + "campaign.base_campaign", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.absolute_top_impression_percentage", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.average_page_views", + "metrics.average_time_on_site", + "metrics.bounce_rate", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cost_per_current_model_attributed_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.current_model_attributed_conversions", + "metrics.current_model_attributed_conversions_value", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.percent_new_visitors", + "metrics.top_impression_percentage", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.value_per_current_model_attributed_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.conversion_adjustment", + "segments.conversion_lag_bucket", + "segments.conversion_or_adjustment_lag_bucket", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.keyword.ad_group_criterion", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", +] +AGE_RANGE_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.base_ad_group", + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group.targeting_setting.target_restrictions", + "ad_group_criterion.age_range.type", + "ad_group_criterion.bid_modifier", + "ad_group_criterion.criterion_id", + "ad_group_criterion.effective_cpc_bid_micros", + "ad_group_criterion.effective_cpc_bid_source", + "ad_group_criterion.effective_cpm_bid_micros", + "ad_group_criterion.effective_cpm_bid_source", + "ad_group_criterion.final_mobile_urls", + "ad_group_criterion.final_urls", + "ad_group_criterion.negative", + "ad_group_criterion.status", + "ad_group_criterion.tracking_url_template", + "ad_group_criterion.url_custom_parameters", + "bidding_strategy.name", + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.bidding_strategy_type", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.week", + "segments.year", +] +AD_GROUP_AUDIENCE_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.base_ad_group", + "ad_group.campaign", + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group.targeting_setting.target_restrictions", + "ad_group.tracking_url_template", + "ad_group.url_custom_parameters", + "ad_group_criterion.bid_modifier", + "ad_group_criterion.criterion_id", + "ad_group_criterion.effective_cpc_bid_micros", + "ad_group_criterion.effective_cpc_bid_source", + "ad_group_criterion.effective_cpm_bid_micros", + "ad_group_criterion.effective_cpm_bid_source", + "ad_group_criterion.final_mobile_urls", + "ad_group_criterion.final_urls", + "ad_group_criterion.status", + "bidding_strategy.name", + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.bidding_strategy_type", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", + "user_list.name", +] +CAMPAIGN_AUDIENCE_PERFORMANCE_REPORT_FIELDS = [ + "bidding_strategy.name", + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.bidding_strategy_type", + "campaign.name", + "campaign.status", + "campaign_criterion.bid_modifier", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", + "user_list.name", +] +CAMPAIGN_PERFORMANCE_REPORT_FIELDS = [ + "bidding_strategy.name", + "campaign.advertising_channel_sub_type", + "campaign.advertising_channel_type", + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.bidding_strategy_type", + "campaign.campaign_budget", + "campaign.end_date", + "campaign.experiment_type", + "campaign.final_url_suffix", + "campaign.id", + "campaign.manual_cpc.enhanced_cpc_enabled", + "campaign.maximize_conversion_value.target_roas", + "campaign.name", + "campaign.percent_cpc.enhanced_cpc_enabled", + "campaign.serving_status", + "campaign.start_date", + "campaign.status", + "campaign.tracking_url_template", + "campaign.url_custom_parameters", + "campaign_budget.amount_micros", + "campaign_budget.explicitly_shared", + "campaign_budget.has_recommended_budget", + "campaign_budget.period", + "campaign_budget.recommended_budget_amount_micros", + "campaign_budget.total_amount_micros", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.absolute_top_impression_percentage", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.average_page_views", + "metrics.average_time_on_site", + "metrics.bounce_rate", + "metrics.clicks", + "metrics.content_budget_lost_impression_share", + "metrics.content_impression_share", + "metrics.content_rank_lost_impression_share", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cost_per_current_model_attributed_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.current_model_attributed_conversions", + "metrics.current_model_attributed_conversions_value", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.invalid_click_rate", + "metrics.invalid_clicks", + "metrics.percent_new_visitors", + "metrics.phone_calls", + "metrics.phone_impressions", + "metrics.phone_through_rate", + "metrics.relative_ctr", + "metrics.search_absolute_top_impression_share", + "metrics.search_budget_lost_absolute_top_impression_share", + "metrics.search_budget_lost_impression_share", + "metrics.search_budget_lost_top_impression_share", + "metrics.search_click_share", + "metrics.search_exact_match_impression_share", + "metrics.search_impression_share", + "metrics.search_rank_lost_absolute_top_impression_share", + "metrics.search_rank_lost_impression_share", + "metrics.search_rank_lost_top_impression_share", + "metrics.search_top_impression_share", + "metrics.top_impression_percentage", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.value_per_current_model_attributed_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.conversion_adjustment", + "segments.conversion_attribution_event_type", + "segments.conversion_lag_bucket", + "segments.conversion_or_adjustment_lag_bucket", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.hour", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", +] +CLICK_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "campaign.id", + "campaign.name", + "campaign.status", + "click_view.ad_group_ad", + "click_view.area_of_interest.city", + "click_view.area_of_interest.country", + "click_view.area_of_interest.metro", + "click_view.area_of_interest.most_specific", + "click_view.area_of_interest.region", + "click_view.campaign_location_target", + "click_view.gclid", + "click_view.location_of_presence.city", + "click_view.location_of_presence.country", + "click_view.location_of_presence.metro", + "click_view.location_of_presence.most_specific", + "click_view.location_of_presence.region", + "click_view.page_number", + "click_view.user_list", + "customer.descriptive_name", + "customer.id", + "metrics.clicks", + "segments.ad_network_type", + "segments.click_type", + "segments.date", + "segments.device", + "segments.month_of_year", + "segments.slot", +] +DISPLAY_KEYWORD_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.base_ad_group", + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group.targeting_setting.target_restrictions", + "ad_group_criterion.criterion_id", + "ad_group_criterion.effective_cpc_bid_micros", + "ad_group_criterion.effective_cpc_bid_source", + "ad_group_criterion.effective_cpm_bid_micros", + "ad_group_criterion.effective_cpm_bid_source", + "ad_group_criterion.effective_cpv_bid_micros", + "ad_group_criterion.effective_cpv_bid_source", + "ad_group_criterion.final_mobile_urls", + "ad_group_criterion.final_urls", + "ad_group_criterion.keyword.text", + "ad_group_criterion.negative", + "ad_group_criterion.status", + "ad_group_criterion.tracking_url_template", + "ad_group_criterion.url_custom_parameters", + "bidding_strategy.name", # bidding_strategy.name must be selected with the resources bidding_strategy and campaign. + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.bidding_strategy_type", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.week", + "segments.year", +] +DISPLAY_TOPICS_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.base_ad_group", + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group.targeting_setting.target_restrictions", + "ad_group_criterion.bid_modifier", + "ad_group_criterion.criterion_id", + "ad_group_criterion.effective_cpc_bid_micros", + "ad_group_criterion.effective_cpc_bid_source", + "ad_group_criterion.effective_cpm_bid_micros", + "ad_group_criterion.effective_cpm_bid_source", + "ad_group_criterion.final_mobile_urls", + "ad_group_criterion.final_urls", + "ad_group_criterion.negative", + "ad_group_criterion.status", + "ad_group_criterion.topic.path", + "ad_group_criterion.topic.topic_constant", + "ad_group_criterion.tracking_url_template", + "ad_group_criterion.url_custom_parameters", + "bidding_strategy.name", # bidding_strategy.name must be selected with the resources bidding_strategy and campaign. + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.bidding_strategy_type", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.week", + "segments.year", +] +EXPANDED_LANDING_PAGE_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "campaign.advertising_channel_type", + "campaign.id", + "campaign.name", + "campaign.status", + "expanded_landing_page_view.expanded_final_url", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.mobile_friendly_clicks_percentage", + "metrics.speed_score", + "metrics.valid_accelerated_mobile_pages_clicks_percentage", + "metrics.value_per_conversion", + "metrics.video_view_rate", + "metrics.video_views", + "segments.ad_network_type", + "segments.click_type", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", +] +GENDER_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.base_ad_group", + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group.targeting_setting.target_restrictions", + "ad_group_criterion.bid_modifier", + "ad_group_criterion.criterion_id", + "ad_group_criterion.effective_cpc_bid_micros", + "ad_group_criterion.effective_cpc_bid_source", + "ad_group_criterion.effective_cpm_bid_micros", + "ad_group_criterion.effective_cpm_bid_source", + "ad_group_criterion.final_mobile_urls", + "ad_group_criterion.final_urls", + "ad_group_criterion.gender.type", + "ad_group_criterion.negative", + "ad_group_criterion.status", + "ad_group_criterion.tracking_url_template", + "ad_group_criterion.url_custom_parameters", + "bidding_strategy.name", + "bidding_strategy.type", + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.week", + "segments.year", +] +GEO_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "geographic_view.country_criterion_id", + "geographic_view.location_type", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.geo_target_city", + "segments.geo_target_metro", + "segments.geo_target_most_specific_location", + "segments.geo_target_region", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.week", + "segments.year", +] +KEYWORDLESS_QUERY_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "dynamic_search_ads_search_term_view.headline", + "dynamic_search_ads_search_term_view.landing_page", + "dynamic_search_ads_search_term_view.search_term", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cpc", + "metrics.average_cpm", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.impressions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.webpage", + "segments.week", + "segments.year", +] +KEYWORDS_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.base_ad_group", + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group_criterion.approval_status", + "ad_group_criterion.criterion_id", + "ad_group_criterion.effective_cpc_bid_micros", + "ad_group_criterion.effective_cpc_bid_source", + "ad_group_criterion.effective_cpm_bid_micros", + "ad_group_criterion.final_mobile_urls", + "ad_group_criterion.final_url_suffix", + "ad_group_criterion.final_urls", + "ad_group_criterion.keyword.match_type", + "ad_group_criterion.keyword.text", + "ad_group_criterion.negative", + "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc", + "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc", + "ad_group_criterion.position_estimates.first_page_cpc_micros", + "ad_group_criterion.position_estimates.first_position_cpc_micros", + "ad_group_criterion.position_estimates.top_of_page_cpc_micros", + "ad_group_criterion.quality_info.creative_quality_score", + "ad_group_criterion.quality_info.post_click_quality_score", + "ad_group_criterion.quality_info.quality_score", + "ad_group_criterion.quality_info.search_predicted_ctr", + "ad_group_criterion.status", + "ad_group_criterion.system_serving_status", + "ad_group_criterion.topic.topic_constant", + "ad_group_criterion.tracking_url_template", + "ad_group_criterion.url_custom_parameters", + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.bidding_strategy_type", + "campaign.id", + "campaign.manual_cpc.enhanced_cpc_enabled", + "campaign.name", + "campaign.percent_cpc.enhanced_cpc_enabled", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.absolute_top_impression_percentage", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.average_page_views", + "metrics.average_time_on_site", + "metrics.bounce_rate", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cost_per_current_model_attributed_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.current_model_attributed_conversions", + "metrics.current_model_attributed_conversions_value", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.historical_creative_quality_score", + "metrics.historical_landing_page_quality_score", + "metrics.historical_quality_score", + "metrics.historical_search_predicted_ctr", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.percent_new_visitors", + "metrics.search_absolute_top_impression_share", + "metrics.search_budget_lost_absolute_top_impression_share", + "metrics.search_budget_lost_top_impression_share", + "metrics.search_exact_match_impression_share", + "metrics.search_impression_share", + "metrics.search_rank_lost_absolute_top_impression_share", + "metrics.search_rank_lost_impression_share", + "metrics.search_rank_lost_top_impression_share", + "metrics.search_top_impression_share", + "metrics.top_impression_percentage", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.value_per_current_model_attributed_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.conversion_adjustment", + "segments.conversion_lag_bucket", + "segments.conversion_or_adjustment_lag_bucket", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", +] +LANDING_PAGE_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "campaign.advertising_channel_type", + "campaign.id", + "campaign.name", + "campaign.status", + "landing_page_view.unexpanded_final_url", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.mobile_friendly_clicks_percentage", + "metrics.speed_score", + "metrics.valid_accelerated_mobile_pages_clicks_percentage", + "metrics.value_per_conversion", + "metrics.video_view_rate", + "metrics.video_views", + "segments.ad_network_type", + "segments.click_type", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", +] +PLACEHOLDER_FEED_ITEM_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group_ad.resource_name", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "feed_item.attribute_values", + "feed_item.end_date_time", + "feed_item.feed", + "feed_item.geo_targeting_restriction", + "feed_item.id", + "feed_item.policy_infos", + "feed_item.start_date_time", + "feed_item.status", + "feed_item.url_custom_parameters", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_view_rate", + "metrics.video_views", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.interaction_on_this_extension", + "segments.month", + "segments.month_of_year", + "segments.placeholder_type", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", +] +PLACEHOLDER_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group_ad.resource_name", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.descriptive_name", + "customer.id", + "feed_placeholder_view.placeholder_type", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.slot", + "segments.week", + "segments.year", +] +PLACEMENT_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.base_ad_group", + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group.targeting_setting.target_restrictions", + "ad_group_criterion.bid_modifier", + "ad_group_criterion.criterion_id", + "ad_group_criterion.effective_cpc_bid_micros", + "ad_group_criterion.effective_cpc_bid_source", + "ad_group_criterion.effective_cpm_bid_micros", + "ad_group_criterion.effective_cpm_bid_source", + "ad_group_criterion.final_mobile_urls", + "ad_group_criterion.final_urls", + "ad_group_criterion.negative", + "ad_group_criterion.placement.url", + "ad_group_criterion.status", + "ad_group_criterion.tracking_url_template", + "ad_group_criterion.url_custom_parameters", + "bidding_strategy.name", + "bidding_strategy.type", + "campaign.base_campaign", + "campaign.bidding_strategy", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.active_view_cpm", + "metrics.active_view_ctr", + "metrics.active_view_impressions", + "metrics.active_view_measurability", + "metrics.active_view_measurable_cost_micros", + "metrics.active_view_measurable_impressions", + "metrics.active_view_viewability", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.gmail_forwards", + "metrics.gmail_saves", + "metrics.gmail_secondary_clicks", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.week", + "segments.year", +] +SEARCH_QUERY_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group_ad.ad.final_urls", + "ad_group_ad.ad.id", + "ad_group_ad.ad.tracking_url_template", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.absolute_top_impression_percentage", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpe", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.top_impression_percentage", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "search_term_view.search_term", + "search_term_view.status", + "segments.ad_network_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.keyword.ad_group_criterion", + "segments.keyword.info.text", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.search_term_match_type", + "segments.week", + "segments.year", +] +SHOPPING_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.descriptive_name", + "customer.id", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cpc", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.impressions", + "metrics.search_absolute_top_impression_share", + "metrics.search_click_share", + "metrics.search_impression_share", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.product_aggregator_id", + "segments.product_bidding_category_level1", + "segments.product_bidding_category_level2", + "segments.product_bidding_category_level3", + "segments.product_bidding_category_level4", + "segments.product_bidding_category_level5", + "segments.product_brand", + "segments.product_channel", + "segments.product_channel_exclusivity", + "segments.product_condition", + "segments.product_country", + "segments.product_custom_attribute0", + "segments.product_custom_attribute1", + "segments.product_custom_attribute2", + "segments.product_custom_attribute3", + "segments.product_custom_attribute4", + "segments.product_item_id", + "segments.product_language", + "segments.product_merchant_id", + "segments.product_store_id", + "segments.product_title", + "segments.product_type_l1", + "segments.product_type_l2", + "segments.product_type_l3", + "segments.product_type_l4", + "segments.product_type_l5", + "segments.quarter", + "segments.week", + "segments.year", +] +USER_LOCATION_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cost", + "metrics.average_cpc", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_from_interactions_rate", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.impressions", + "metrics.interaction_event_types", + "metrics.interaction_rate", + "metrics.interactions", + "metrics.value_per_all_conversions", + "metrics.value_per_conversion", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.geo_target_city", + "segments.geo_target_metro", + "segments.geo_target_most_specific_location", + "segments.geo_target_region", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.week", + "segments.year", + "user_location_view.country_criterion_id", + "user_location_view.targeting_location", +] +VIDEO_PERFORMANCE_REPORT_FIELDS = [ + "ad_group.id", + "ad_group.name", + "ad_group.status", + "ad_group_ad.ad.id", + "ad_group_ad.status", + "campaign.id", + "campaign.name", + "campaign.status", + "customer.currency_code", + "customer.descriptive_name", + "customer.descriptive_name", + "customer.id", + "customer.time_zone", + "metrics.all_conversions", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_from_interactions_rate", + "metrics.all_conversions_value", + "metrics.average_cpm", + "metrics.average_cpv", + "metrics.clicks", + "metrics.conversions", + "metrics.conversions_value", + "metrics.cost_micros", + "metrics.cost_per_all_conversions", + "metrics.cost_per_conversion", + "metrics.cross_device_conversions", + "metrics.ctr", + "metrics.engagement_rate", + "metrics.engagements", + "metrics.impressions", + "metrics.value_per_all_conversions", + "metrics.video_quartile_p100_rate", + "metrics.video_quartile_p25_rate", + "metrics.video_quartile_p50_rate", + "metrics.video_quartile_p75_rate", + "metrics.video_view_rate", + "metrics.video_views", + "metrics.view_through_conversions", + "segments.ad_network_type", + "segments.click_type", + "segments.conversion_action", + "segments.conversion_action_category", + "segments.conversion_action_name", + "segments.date", + "segments.day_of_week", + "segments.device", + "segments.external_conversion_source", + "segments.month", + "segments.month_of_year", + "segments.quarter", + "segments.week", + "segments.year", + "video.channel_id", + "video.duration_millis", + "video.id", + "video.title", +] diff --git a/tap_google_ads/reports.py b/tap_google_ads/reports.py deleted file mode 100644 index 2a02399..0000000 --- a/tap_google_ads/reports.py +++ /dev/null @@ -1,445 +0,0 @@ -from collections import defaultdict -import json - -import singer -from singer import Transformer - -from google.protobuf.json_format import MessageToJson - -from . import report_definitions - -LOGGER = singer.get_logger() - -API_VERSION = "v9" - -CORE_STREAMS = [ - "customer", - "ad_group", - "ad_group_ad", - "campaign", - "bidding_strategy", - "accessible_bidding_strategy", - "campaign_budget", -] - - -def flatten(obj): - """Given an `obj` like - - {"a" : {"b" : "c"}, - "d": "e"} - - return - - {"a.b": "c", - "d": "e"} - """ - new_obj = {} - for key, value in obj.items(): - if isinstance(value, dict): - for sub_key, sub_value in flatten(value).items(): - new_obj[f"{key}.{sub_key}"] = sub_value - else: - new_obj[key] = value - return new_obj - - -def make_field_names(resource_name, fields): - transformed_fields = [] - for field in fields: - pieces = field.split("_") - front = "_".join(pieces[:-1]) - back = pieces[-1] - - if '.' in field: - transformed_fields.append(f"{resource_name}.{field}") - elif front in CORE_STREAMS and field.endswith('_id'): - transformed_fields.append(f"{front}.{back}") - else: - transformed_fields.append(f"{resource_name}.{field}") - return transformed_fields - - -def transform_keys(resource_name, flattened_obj): - transformed_obj = {} - - for field, value in flattened_obj.items(): - resource_matches = field.startswith(resource_name + ".") - is_id_field = field.endswith(".id") - - if resource_matches: - new_field_name = ".".join(field.split(".")[1:]) - elif is_id_field: - new_field_name = field.replace(".", "_") - else: - new_field_name = field - - assert new_field_name not in transformed_obj - transformed_obj[new_field_name] = value - - return transformed_obj - -class BaseStream: - def sync(self, sdk_client, customer, stream): - gas = sdk_client.get_service("GoogleAdsService", version=API_VERSION) - resource_name = self.google_ads_resources_name[0] - stream_name = stream["stream"] - stream_mdata = stream["metadata"] - selected_fields = [] - for mdata in stream_mdata: - if ( - mdata["breadcrumb"] - and mdata["metadata"].get("selected") - and mdata["metadata"].get("inclusion") == "available" - ): - selected_fields.append(mdata["breadcrumb"][1]) - - google_field_names = make_field_names(resource_name, selected_fields) - query = f"SELECT {','.join(google_field_names)} FROM {resource_name}" - response = gas.search(query=query, customer_id=customer["customerId"]) - with Transformer() as transformer: - json_response = [ - json.loads(MessageToJson(x, preserving_proto_field_name=True)) - for x in response - ] - for obj in json_response: - flattened_obj = flatten(obj) - transformed_obj = transform_keys(resource_name, flattened_obj) - record = transformer.transform(transformed_obj, stream["schema"]) - singer.write_record(stream_name, record) - - def add_extra_fields(self, resource_schema): - """This function should add fields to `field_exclusions`, `schema`, and - `behavior` that are not covered by Google's resource_schema - """ - - def extract_field_information(self, resource_schema): - self.field_exclusions = defaultdict(set) - self.schema = {} - self.behavior = {} - self.selectable = {} - - for resource_name in self.google_ads_resources_name: - - # field_exclusions step - fields = resource_schema[resource_name]["fields"] - for field_name, field in fields.items(): - if field_name in self.fields: - self.field_exclusions[field_name].update( - field["incompatible_fields"] - ) - - self.schema[field_name] = field["field_details"]["json_schema"] - - self.behavior[field_name] = field["field_details"]["category"] - - self.selectable[field_name] = field["field_details"]["selectable"] - self.add_extra_fields(resource_schema) - self.field_exclusions = {k: list(v) for k, v in self.field_exclusions.items()} - - def __init__(self, fields, google_ads_resource_name, resource_schema, primary_keys): - self.fields = fields - self.google_ads_resources_name = google_ads_resource_name - self.primary_keys = primary_keys - self.extract_field_information(resource_schema) - - -class AdGroupPerformanceReport(BaseStream): - def add_extra_fields(self, resource_schema): - # from the resource ad_group_ad_label - field_name = "label.resource_name" - # for field_name in []: - self.field_exclusions[field_name] = {} - self.schema[field_name] = {"type": ["null", "string"]} - self.behavior[field_name] = "ATTRIBUTE" - - -class AdPerformanceReport(BaseStream): - def add_extra_fields(self, resource_schema): - # from the resource ad_group_ad_label - for field_name in ["label.resource_name", "label.name"]: - self.field_exclusions[field_name] = {} - self.schema[field_name] = {"type": ["null", "string"]} - self.behavior[field_name] = "ATTRIBUTE" - - for field_name in [ - "ad_group_criterion.negative", - ]: - self.field_exclusions[field_name] = {} - self.schema[field_name] = {"type": ["null", "boolean"]} - self.behavior[field_name] = "ATTRIBUTE" - - -class AudiencePerformanceReport(BaseStream): - "hi" - # COMMENT FROM GOOGLE - #'bidding_strategy.name must be selected withy the resources bidding_strategy or campaign.', - - # We think this means - # `SELECT bidding_strategy.name from bidding_strategy` - # Not sure how this applies to the campaign resource - - # COMMENT FROM GOOGLE - # 'campaign.bidding_strategy.type must be selected withy the resources bidding_strategy or campaign.' - - # We think this means - # `SELECT bidding_strategy.type from bidding_strategy` - - # `SELECT campaign.bidding_strategy_type from campaign` - - # 'user_list.name' is a "Segmenting resource" - # `select user_list.name from ` - -class CampaignPerformanceReport(BaseStream): - # TODO: The sync needs to select from campaign_criterion if campaign_criterion.device.type is selected - # TODO: The sync needs to select from campaign_label if label.resource_name - def add_extra_fields(self, resource_schema): - for field_name in [ - "campaign_criterion.device.type", - "label.resource_name", - ]: - self.field_exclusions[field_name] = set() - self.schema[field_name] = {"type": ["null", "string"]} - self.behavior[field_name] = "ATTRIBUTE" - - -class DisplayKeywordPerformanceReport(BaseStream): - # TODO: The sync needs to select from bidding_strategy and/or campaign if bidding_strategy.name is selected - def add_extra_fields(self, resource_schema): - for field_name in [ - "bidding_strategy.name", - ]: - self.field_exclusions[field_name] = resource_schema[ - self.google_ads_resources_name[0] - ]["fields"][field_name]["incompatible_fields"] - self.schema[field_name] = {"type": ["null", "string"]} - self.behavior[field_name] = "SEGMENT" - - -class GeoPerformanceReport(BaseStream): - # TODO: The sync needs to select from bidding_strategy and/or campaign if bidding_strategy.name is selected - def add_extra_fields(self, resource_schema): - for resource_name in self.google_ads_resources_name: - for field_name in [ - "country_criterion_id", - ]: - full_field_name = f"{resource_name}.{field_name}" - self.field_exclusions[full_field_name] = ( - resource_schema[resource_name]["fields"][full_field_name][ - "incompatible_fields" - ] - or set() - ) - self.schema[full_field_name] = {"type": ["null", "string"]} - self.behavior[full_field_name] = "ATTRIBUTE" - - -class KeywordsPerformanceReport(BaseStream): - # TODO: The sync needs to select from ad_group_label if label.name is selected - # TODO: The sync needs to select from ad_group_label if label.resource_name is selected - def add_extra_fields(self, resource_schema): - for field_name in [ - "label.resource_name", - "label.name", - ]: - self.field_exclusions[field_name] = set() - self.schema[field_name] = {"type": ["null", "string"]} - self.behavior[field_name] = "ATTRIBUTE" - - -class PlaceholderFeedItemReport(BaseStream): - # TODO: The sync needs to select from feed_item_target if feed_item_target.device is selected - # TODO: The sync needs to select from feed_item if feed_item.policy_infos is selected - def add_extra_fields(self, resource_schema): - for field_name in ["feed_item_target.device", "feed_item.policy_infos"]: - self.field_exclusions[field_name] = set() - self.schema[field_name] = {"type": ["null", "string"]} - self.behavior[field_name] = "ATTRIBUTE" - - -def initialize_core_streams(resource_schema): - return { - "accounts": BaseStream( - report_definitions.ACCOUNT_FIELDS, - ["customer"], - resource_schema, - ["customer.id"], - ), - "ad_groups": BaseStream( - report_definitions.AD_GROUP_FIELDS, - ["ad_group"], - resource_schema, - ["ad_group.id"], - ), - "ads": BaseStream( - report_definitions.AD_GROUP_AD_FIELDS, - ["ad_group_ad"], - resource_schema, - ["ad_group_ad.ad.id"], - ), - "campaigns": BaseStream( - report_definitions.CAMPAIGN_FIELDS, - ["campaign"], - resource_schema, - ["campaign.id"], - ), - "bidding_strategies": BaseStream( - report_definitions.BIDDING_STRATEGY_FIELDS, - ["bidding_strategy"], - resource_schema, - ["bidding_strategy.id"], - ), - "accessible_bidding_strategies": BaseStream( - report_definitions.ACCESSIBLE_BIDDING_STRATEGY_FIELDS, - ["accessible_bidding_strategy"], - resource_schema, - ["accessible_bidding_strategy.id"], - ), - "campaign_budgets": BaseStream( - report_definitions.CAMPAIGN_BUDGET_FIELDS, - ["campaign_budget"], - resource_schema, - ["campaign_budget.id"], - ), - } - - -def initialize_reports(resource_schema): - return { - "account_performance_report": BaseStream( - report_definitions.ACCOUNT_PERFORMANCE_REPORT_FIELDS, - ["customer"], - resource_schema, - ["customer.id"], - ), - # TODO: This needs to link with ad_group_ad_label - "adgroup_performance_report": AdGroupPerformanceReport( - report_definitions.ADGROUP_PERFORMANCE_REPORT_FIELDS, - ["ad_group"], - resource_schema, - ["ad_group.id"], - ), - "ad_performance_report": AdPerformanceReport( - report_definitions.AD_PERFORMANCE_REPORT_FIELDS, - ["ad_group_ad"], - resource_schema, - ["ad_group_ad.ad.id"], - ), - "age_range_performance_report": BaseStream( - report_definitions.AGE_RANGE_PERFORMANCE_REPORT_FIELDS, - ["age_range_view"], - resource_schema, - ["ad_group_criterion.criterion_id"], - ), - "audience_performance_report": AudiencePerformanceReport( - report_definitions.AUDIENCE_PERFORMANCE_REPORT_FIELDS, - ["campaign_audience_view", "ad_group_audience_view"], - resource_schema, - ["ad_group_criterion.criterion_id"], - ), - "call_metrics_call_details_report": BaseStream( - report_definitions.CALL_METRICS_CALL_DETAILS_REPORT_FIELDS, - ["call_view"], - resource_schema, - [""], - ), - "campaign_performance_report": CampaignPerformanceReport( - report_definitions.CAMPAIGN_PERFORMANCE_REPORT_FIELDS, - ["campaign"], - resource_schema, - [""], - ), - "click_performance_report": BaseStream( - report_definitions.CLICK_PERFORMANCE_REPORT_FIELDS, - ["click_view"], - resource_schema, - [""], - ), - "display_keyword_performance_report": DisplayKeywordPerformanceReport( - report_definitions.DISPLAY_KEYWORD_PERFORMANCE_REPORT_FIELDS, - ["display_keyword_view"], - resource_schema, - ["ad_group_criterion.criterion_id"], - ), - "display_topics_performance_report": DisplayKeywordPerformanceReport( - report_definitions.DISPLAY_TOPICS_PERFORMANCE_REPORT_FIELDS, - ["topic_view"], - resource_schema, - [""], - ), - "gender_performance_report": BaseStream( - report_definitions.GENDER_PERFORMANCE_REPORT_FIELDS, - ["gender_view"], - resource_schema, - [""], - ), - "geo_performance_report": GeoPerformanceReport( - report_definitions.GEO_PERFORMANCE_REPORT_FIELDS, - ["geographic_view", "user_location_view"], - resource_schema, - [""], - ), - "keywordless_query_report": BaseStream( - report_definitions.KEYWORDLESS_QUERY_REPORT_FIELDS, - ["dynamic_search_ads_search_term_view"], - resource_schema, - [""], - ), - "keywords_performance_report": KeywordsPerformanceReport( - report_definitions.KEYWORDS_PERFORMANCE_REPORT_FIELDS, - ["keyword_view"], - resource_schema, - [""], - ), - "placeholder_feed_item_report": PlaceholderFeedItemReport( - report_definitions.PLACEHOLDER_FEED_ITEM_REPORT_FIELDS, - ["feed_item", "feed_item_target"], - resource_schema, - [""], - ), - "placeholder_report": BaseStream( - report_definitions.PLACEHOLDER_REPORT_FIELDS, - ["feed_placeholder_view"], - resource_schema, - [""], - ), - "placement_performance_report": BaseStream( - report_definitions.PLACEMENT_PERFORMANCE_REPORT_FIELDS, - ["managed_placement_view"], - resource_schema, - [""], - ), - "search_query_performance_report": BaseStream( - report_definitions.SEARCH_QUERY_PERFORMANCE_REPORT_FIELDS, - ["search_term_view"], - resource_schema, - [""], - ), - "shopping_performance_report": BaseStream( - report_definitions.SHOPPING_PERFORMANCE_REPORT_FIELDS, - ["shopping_performance_view"], - resource_schema, - [""], - ), - "video_performance_report": BaseStream( - report_definitions.VIDEO_PERFORMANCE_REPORT_FIELDS, - ["video"], - resource_schema, - [""], - ), - # "automatic_placements_performance_report": BaseStream(report_definitions.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT_FIELDS, ["group_placement_view"], resource_schema), - # "bid_goal_performance_report": BaseStream(report_definitions.BID_GOAL_PERFORMANCE_REPORT_FIELDS, ["bidding_strategy"], resource_schema), - # "budget_performance_report": BaseStream(report_definitions.BUDGET_PERFORMANCE_REPORT_FIELDS, ["campaign_budget"], resource_schema), - # "campaign_ad_schedule_target_report": BaseStream(report_definitions.CAMPAIGN_AD_SCHEDULE_TARGET_REPORT_FIELDS, ["ad_schedule_view"], resource_schema), - # "campaign_criteria_report": BaseStream(report_definitions.CAMPAIGN_CRITERIA_REPORT_FIELDS, ["campaign_criterion"], resource_schema), - # "campaign_location_target_report": BaseStream(report_definitions.CAMPAIGN_LOCATION_TARGET_REPORT_FIELDS, ["location_view"], resource_schema), - # "campaign_shared_set_report": BaseStream(report_definitions.CAMPAIGN_SHARED_SET_REPORT_FIELDS, ["campaign_shared_set"], resource_schema), - # "label_report": BaseStream(report_definitions.LABEL_REPORT_FIELDS, ["label"], resource_schema), - # "landing_page_report": BaseStream(report_definitions.LANDING_PAGE_REPORT_FIELDS, ["landing_page_view", "expanded_landing_page_view"], resource_schema), - # "paid_organic_query_report": BaseStream(report_definitions.PAID_ORGANIC_QUERY_REPORT_FIELDS, ["paid_organic_search_term_view"], resource_schema), - # "parental_status_performance_report": BaseStream(report_definitions.PARENTAL_STATUS_PERFORMANCE_REPORT_FIELDS, ["parental_status_view"], resource_schema), - # "product_partition_report": BaseStream(report_definitions.PRODUCT_PARTITION_REPORT_FIELDS, ["product_group_view"], resource_schema), - # "shared_set_criteria_report": BaseStream(report_definitions.SHARED_SET_CRITERIA_REPORT_FIELDS, ["shared_criterion"], resource_schema), - # "url_performance_report": BaseStream(report_definitions.URL_PERFORMANCE_REPORT_FIELDS, ["detail_placement_view"], resource_schema), - # "user_ad_distance_report": BaseStream(report_definitions.USER_AD_DISTANCE_REPORT_FIELDS, ["distance_view"], resource_schema), - } diff --git a/tap_google_ads/streams.py b/tap_google_ads/streams.py new file mode 100644 index 0000000..09c3aad --- /dev/null +++ b/tap_google_ads/streams.py @@ -0,0 +1,620 @@ +from collections import defaultdict +import json +import hashlib +from datetime import timedelta +import singer +from singer import Transformer +from singer import utils +from google.protobuf.json_format import MessageToJson +from . import report_definitions + +LOGGER = singer.get_logger() + +API_VERSION = "v9" + +REPORTS_WITH_90_DAY_MAX = frozenset( + [ + "click_performance_report", + ] +) + +DEFAULT_CONVERSION_WINDOW = 30 + + +def create_nested_resource_schema(resource_schema, fields): + new_schema = { + "type": ["null", "object"], + "properties": {} + } + + for field in fields: + walker = new_schema["properties"] + paths = field.split(".") + last_path = paths[-1] + for path in paths[:-1]: + if path not in walker: + walker[path] = { + "type": ["null", "object"], + "properties": {} + } + walker = walker[path]["properties"] + if last_path not in walker: + json_schema = resource_schema[field]["json_schema"] + walker[last_path] = json_schema + return new_schema + + +def get_selected_fields(stream_mdata): + selected_fields = set() + for mdata in stream_mdata: + if mdata["breadcrumb"]: + inclusion = mdata["metadata"].get("inclusion") + selected = mdata["metadata"].get("selected") + if utils.should_sync_field(inclusion, selected) and mdata["breadcrumb"][1] != "_sdc_record_hash": + selected_fields.update(mdata["metadata"]["tap-google-ads.api-field-names"]) + + return selected_fields + + +def create_core_stream_query(resource_name, selected_fields): + core_query = f"SELECT {','.join(selected_fields)} FROM {resource_name}" + return core_query + + +def create_report_query(resource_name, selected_fields, query_date): + + format_str = "%Y-%m-%d" + query_date = utils.strftime(query_date, format_str=format_str) + report_query = f"SELECT {','.join(selected_fields)} FROM {resource_name} WHERE segments.date = '{query_date}'" + + return report_query + + +def generate_hash(record, metadata): + metadata = singer.metadata.to_map(metadata) + fields_to_hash = {} + for key, val in record.items(): + if metadata[("properties", key)]["behavior"] != "METRIC": + fields_to_hash[key] = val + + hash_source_data = {key: fields_to_hash[key] for key in sorted(fields_to_hash)} + hash_bytes = json.dumps(hash_source_data).encode("utf-8") + return hashlib.sha256(hash_bytes).hexdigest() + + +class BaseStream: # pylint: disable=too-many-instance-attributes + + def __init__(self, fields, google_ads_resource_names, resource_schema, primary_keys): + self.fields = fields + self.google_ads_resource_names = google_ads_resource_names + self.primary_keys = primary_keys + + self.extract_field_information(resource_schema) + + self.create_full_schema(resource_schema) + self.set_stream_schema() + self.format_field_names() + + self.build_stream_metadata() + + + def extract_field_information(self, resource_schema): + self.field_exclusions = defaultdict(set) + self.schema = {} + self.behavior = {} + self.selectable = {} + + for resource_name in self.google_ads_resource_names: + + # field_exclusions step + fields = resource_schema[resource_name]["fields"] + for field_name, field in fields.items(): + if field_name in self.fields: + self.field_exclusions[field_name].update( + field["incompatible_fields"] + ) + + self.schema[field_name] = field["field_details"]["json_schema"] + + self.behavior[field_name] = field["field_details"]["category"] + + self.selectable[field_name] = field["field_details"]["selectable"] + self.add_extra_fields(resource_schema) + self.field_exclusions = {k: list(v) for k, v in self.field_exclusions.items()} + + def add_extra_fields(self, resource_schema): + """This function should add fields to `field_exclusions`, `schema`, and + `behavior` that are not covered by Google's resource_schema + """ + + def create_full_schema(self, resource_schema): + google_ads_name = self.google_ads_resource_names[0] + self.resource_object = resource_schema[google_ads_name] + self.resource_fields = self.resource_object["fields"] + self.full_schema = create_nested_resource_schema(resource_schema, self.resource_fields) + + def set_stream_schema(self): + google_ads_name = self.google_ads_resource_names[0] + self.stream_schema = self.full_schema["properties"][google_ads_name] + + def format_field_names(self): + """This function does two things: + 1. Appends a `resource_name` to an id field if it is the id of an attributed resource + 2. Lifts subfields of `ad_group_ad.ad` into `ad_group_ad` + """ + for resource_name, schema in self.full_schema["properties"].items(): + # ads stream is special since all of the ad fields are nested under ad_group_ad.ad + # we need to bump the fields up a level so they are selectable + if resource_name == "ad_group_ad": + for ad_field_name, ad_field_schema in self.full_schema["properties"]["ad_group_ad"]["properties"]["ad"]["properties"].items(): + self.stream_schema["properties"][ad_field_name] = ad_field_schema + self.stream_schema["properties"].pop("ad") + + if ( + resource_name not in {"metrics", "segments"} + and resource_name not in self.google_ads_resource_names + ): + self.stream_schema["properties"][resource_name + "_id"] = schema["properties"]["id"] + + def build_stream_metadata(self): + self.stream_metadata = { + (): { + "inclusion": "available", + "forced-replication-method": "FULL_TABLE", + "table-key-properties": self.primary_keys, + } + } + + for field, props in self.resource_fields.items(): + resource_matches = field.startswith(self.resource_object["name"] + ".") + is_id_field = field.endswith(".id") + + if is_id_field or (props["field_details"]["category"] == "ATTRIBUTE" and resource_matches): + # Transform the field name to match the schema + # Special case for ads since they are nested under ad_group_ad and + # we have to bump them up a level + if field.startswith("ad_group_ad.ad."): + field = field.split(".")[2] + else: + if resource_matches: + field = field.split(".")[1] + elif is_id_field: + field = field.replace(".", "_") + + if ("properties", field) not in self.stream_metadata: + # Base metadata for every field + self.stream_metadata[("properties", field)] = { + "fieldExclusions": props["incompatible_fields"], + "behavior": props["field_details"]["category"], + } + + # Add inclusion metadata + # Foreign keys are automatically included and they are all id fields + if field in self.primary_keys or field in {'customer_id', 'ad_group_id', 'campaign_id'}: + inclusion = "automatic" + elif props["field_details"]["selectable"]: + inclusion = "available" + else: + # inclusion = "unsupported" + continue + self.stream_metadata[("properties", field)]["inclusion"] = inclusion + + # Save the full field name for sync code to use + full_name = props["field_details"]["name"] + if "tap-google-ads.api-field-names" not in self.stream_metadata[("properties", field)]: + self.stream_metadata[("properties", field)]["tap-google-ads.api-field-names"] = [] + + if props["field_details"]["selectable"]: + self.stream_metadata[("properties", field)]["tap-google-ads.api-field-names"].append(full_name) + + def transform_keys(self, obj): + """This function does a few things with Google's response for sync queries: + 1) checks an object's fields to see if they're for the current resource + 2) if they are, keep the fields in transformed_obj with no modifications + 3) if they are not, append a foreign key to the transformed_obj using the id value + 4) if the resource is ad_group_ad, pops ad fields up to the ad_group_ad level + + We've seen API responses where Google returns `type_` when the + field we ask for is `type`, so we transfrom the key-value pair + `"type_": X` to `"type": X` + """ + target_resource_name = self.google_ads_resource_names[0] + transformed_obj = {} + + for resource_name, value in obj.items(): + resource_matches = target_resource_name == resource_name + + if resource_matches: + transformed_obj.update(value) + else: + transformed_obj[f"{resource_name}_id"] = value["id"] + + if resource_name == "ad_group_ad": + transformed_obj.update(value["ad"]) + transformed_obj.pop("ad") + + if "type_" in transformed_obj: + transformed_obj["type"] = transformed_obj.pop("type_") + + return transformed_obj + + def sync(self, sdk_client, customer, stream, config, state): # pylint: disable=unused-argument + gas = sdk_client.get_service("GoogleAdsService", version=API_VERSION) + resource_name = self.google_ads_resource_names[0] + stream_name = stream["stream"] + stream_mdata = stream["metadata"] + selected_fields = get_selected_fields(stream_mdata) + state = singer.set_currently_syncing(state, stream_name) + LOGGER.info(f"Selected fields for stream {stream_name}: {selected_fields}") + + query = create_core_stream_query(resource_name, selected_fields) + response = gas.search(query=query, customer_id=customer["customerId"]) + with Transformer() as transformer: + # Pages are fetched automatically while iterating through the response + for message in response: + json_message = json.loads(MessageToJson(message, preserving_proto_field_name=True)) + transformed_obj = self.transform_keys(json_message) + record = transformer.transform(transformed_obj, stream["schema"], singer.metadata.to_map(stream_mdata)) + + singer.write_record(stream_name, record) + + state = singer.bookmarks.set_currently_syncing(state, None) + + +def get_query_date(start_date, bookmark, conversion_window_date): + """Return a date within the conversion window and after start date + + All inputs are datetime strings. + NOTE: `bookmark` may be None""" + if not bookmark: + return singer.utils.strptime_to_utc(start_date) + else: + query_date = min(bookmark, max(start_date, conversion_window_date)) + return singer.utils.strptime_to_utc(query_date) + + +class ReportStream(BaseStream): + def create_full_schema(self, resource_schema): + google_ads_name = self.google_ads_resource_names[0] + self.resource_object = resource_schema[google_ads_name] + self.resource_fields = self.resource_object["fields"] + self.full_schema = create_nested_resource_schema(resource_schema, self.fields) + + def set_stream_schema(self): + self.stream_schema = { + "type": ["null", "object"], + "is_report": True, + "properties": { + "_sdc_record_hash": {"type": "string"} + }, + } + + def format_field_names(self): + """This function does two things right now: + 1. Appends a `resource_name` to a field name if the field is in an attributed resource + 2. Lifts subfields of `ad_group_ad.ad` into `ad_group_ad` + """ + for resource_name, schema in self.full_schema["properties"].items(): + for field_name, data_type in schema["properties"].items(): + # Ensure that attributed resource fields have the resource name as a prefix, eg campaign_id under the ad_groups stream + if resource_name not in {"metrics", "segments"} and resource_name not in self.google_ads_resource_names: + self.stream_schema["properties"][f"{resource_name}_{field_name}"] = data_type + # Move ad_group_ad.ad.x fields up a level in the schema (ad_group_ad.ad.x -> ad_group_ad.x) + elif resource_name == "ad_group_ad" and field_name == "ad": + for ad_field_name, ad_field_schema in data_type["properties"].items(): + self.stream_schema["properties"][ad_field_name] = ad_field_schema + else: + self.stream_schema["properties"][field_name] = data_type + + def build_stream_metadata(self): + self.stream_metadata = { + (): { + "inclusion": "available", + "table-key-properties": ["_sdc_record_hash"], + "forced-replication-method": "INCREMENTAL", + "valid-replication-keys": ["date"] + }, + ("properties", "_sdc_record_hash"): { + "inclusion": "automatic" + }, + } + for report_field in self.fields: + # Transform the field name to match the schema + is_metric_or_segment = report_field.startswith("metrics.") or report_field.startswith("segments.") + if (not is_metric_or_segment + and report_field.split(".")[0] not in self.google_ads_resource_names + ): + transformed_field_name = "_".join(report_field.split(".")[:2]) + # Transform ad_group_ad.ad.x fields to just x to reflect ad_group_ads schema + elif report_field.startswith("ad_group_ad.ad."): + transformed_field_name = report_field.split(".")[2] + else: + transformed_field_name = report_field.split(".")[1] + + # Base metadata for every field + if ("properties", transformed_field_name) not in self.stream_metadata: + self.stream_metadata[("properties", transformed_field_name)] = { + "fieldExclusions": [], + "behavior": self.behavior[report_field], + } + + # Transform field exclusion names so they match the schema + for field_name in self.field_exclusions[report_field]: + is_metric_or_segment = field_name.startswith("metrics.") or field_name.startswith("segments.") + if (not is_metric_or_segment + and field_name.split(".")[0] not in self.google_ads_resource_names + ): + new_field_name = field_name.replace(".", "_") + else: + new_field_name = field_name.split(".")[1] + + self.stream_metadata[("properties", transformed_field_name)]["fieldExclusions"].append(new_field_name) + + # Add inclusion metadata + if self.behavior[report_field]: + inclusion = "available" + if report_field == "segments.date": + inclusion = "automatic" + else: + inclusion = "unsupported" + self.stream_metadata[("properties", transformed_field_name)]["inclusion"] = inclusion + + # Save the full field name for sync code to use + if "tap-google-ads.api-field-names" not in self.stream_metadata[("properties", transformed_field_name)]: + self.stream_metadata[("properties", transformed_field_name)]["tap-google-ads.api-field-names"] = [] + + self.stream_metadata[("properties", transformed_field_name)]["tap-google-ads.api-field-names"].append(report_field) + + def transform_keys(self, obj): + transformed_obj = {} + + for resource_name, value in obj.items(): + if resource_name == "ad_group_ad": + transformed_obj.update(value["ad"]) + else: + transformed_obj.update(value) + + if "type_" in transformed_obj: + transformed_obj["type"] = transformed_obj.pop("type_") + + return transformed_obj + + def sync(self, sdk_client, customer, stream, config, state): + gas = sdk_client.get_service("GoogleAdsService", version=API_VERSION) + resource_name = self.google_ads_resource_names[0] + stream_name = stream["stream"] + stream_mdata = stream["metadata"] + selected_fields = get_selected_fields(stream_mdata) + replication_key = "date" + state = singer.set_currently_syncing(state, stream_name) + conversion_window = timedelta( + days=int(config.get("conversion_window") or DEFAULT_CONVERSION_WINDOW) + ) + conversion_window_date = utils.now() - conversion_window + + query_date = get_query_date( + start_date=config["start_date"], + bookmark=singer.get_bookmark(state, stream_name, replication_key), + conversion_window_date=singer.utils.strftime(conversion_window_date) + ) + end_date = utils.now() + + if stream_name in REPORTS_WITH_90_DAY_MAX: + cutoff = end_date - timedelta(days=90) + query_date = max(query_date, cutoff) + if query_date == cutoff: + LOGGER.info(f"Stream: {stream_name} supports only 90 days of data. Setting query date to {utils.strftime(query_date, '%Y-%m-%d')}.") + + LOGGER.info(f"Selected fields for stream {stream_name}: {selected_fields}") + singer.write_state(state) + + if selected_fields == {'segments.date'}: + raise Exception(f"Selected fields is currently limited to {', '.join(selected_fields)}. Please select at least one attribute and metric in order to replicate {stream_name}.") + + while query_date < end_date: + query = create_report_query(resource_name, selected_fields, query_date) + LOGGER.info(f"Requesting {stream_name} data for {utils.strftime(query_date, '%Y-%m-%d')}.") + response = gas.search(query=query, customer_id=customer["customerId"]) + + with Transformer() as transformer: + # Pages are fetched automatically while iterating through the response + for message in response: + json_message = json.loads(MessageToJson(message, preserving_proto_field_name=True)) + transformed_obj = self.transform_keys(json_message) + record = transformer.transform(transformed_obj, stream["schema"]) + record["_sdc_record_hash"] = generate_hash(record, stream_mdata) + + singer.write_record(stream_name, record) + + singer.write_bookmark(state, stream_name, replication_key, utils.strftime(query_date)) + + singer.write_state(state) + + query_date += timedelta(days=1) + + state = singer.bookmarks.set_currently_syncing(state, None) + singer.write_state(state) + + +def initialize_core_streams(resource_schema): + return { + "accounts": BaseStream( + report_definitions.ACCOUNT_FIELDS, + ["customer"], + resource_schema, + ["id"], + ), + "ad_groups": BaseStream( + report_definitions.AD_GROUP_FIELDS, + ["ad_group"], + resource_schema, + ["id"], + ), + "ads": BaseStream( + report_definitions.AD_GROUP_AD_FIELDS, + ["ad_group_ad"], + resource_schema, + ["id"], + ), + "campaigns": BaseStream( + report_definitions.CAMPAIGN_FIELDS, + ["campaign"], + resource_schema, + ["id"], + ), + "bidding_strategies": BaseStream( + report_definitions.BIDDING_STRATEGY_FIELDS, + ["bidding_strategy"], + resource_schema, + ["id"], + ), + "accessible_bidding_strategies": BaseStream( + report_definitions.ACCESSIBLE_BIDDING_STRATEGY_FIELDS, + ["accessible_bidding_strategy"], + resource_schema, + ["id"], + ), + "campaign_budgets": BaseStream( + report_definitions.CAMPAIGN_BUDGET_FIELDS, + ["campaign_budget"], + resource_schema, + ["id"], + ), + } + + +def initialize_reports(resource_schema): + return { + "account_performance_report": ReportStream( + report_definitions.ACCOUNT_PERFORMANCE_REPORT_FIELDS, + ["customer"], + resource_schema, + ["_sdc_record_hash"], + ), + "adgroup_performance_report": ReportStream( + report_definitions.ADGROUP_PERFORMANCE_REPORT_FIELDS, + ["ad_group"], + resource_schema, + ["_sdc_record_hash"], + ), + "ad_performance_report": ReportStream( + report_definitions.AD_PERFORMANCE_REPORT_FIELDS, + ["ad_group_ad"], + resource_schema, + ["_sdc_record_hash"], + ), + "age_range_performance_report": ReportStream( + report_definitions.AGE_RANGE_PERFORMANCE_REPORT_FIELDS, + ["age_range_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "audience_performance_report": ReportStream( + report_definitions.AD_GROUP_AUDIENCE_PERFORMANCE_REPORT_FIELDS, + ["ad_group_audience_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "campaign_performance_report": ReportStream( + report_definitions.CAMPAIGN_PERFORMANCE_REPORT_FIELDS, + ["campaign"], + resource_schema, + ["_sdc_record_hash"], + ), + "click_performance_report": ReportStream( + report_definitions.CLICK_PERFORMANCE_REPORT_FIELDS, + ["click_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "display_keyword_performance_report": ReportStream( + report_definitions.DISPLAY_KEYWORD_PERFORMANCE_REPORT_FIELDS, + ["display_keyword_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "display_topics_performance_report": ReportStream( + report_definitions.DISPLAY_TOPICS_PERFORMANCE_REPORT_FIELDS, + ["topic_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "expanded_landing_page_report": ReportStream( + report_definitions.EXPANDED_LANDING_PAGE_REPORT_FIELDS, + ["expanded_landing_page_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "gender_performance_report": ReportStream( + report_definitions.GENDER_PERFORMANCE_REPORT_FIELDS, + ["gender_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "geo_performance_report": ReportStream( + report_definitions.GEO_PERFORMANCE_REPORT_FIELDS, + ["geographic_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "keywordless_query_report": ReportStream( + report_definitions.KEYWORDLESS_QUERY_REPORT_FIELDS, + ["dynamic_search_ads_search_term_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "keywords_performance_report": ReportStream( + report_definitions.KEYWORDS_PERFORMANCE_REPORT_FIELDS, + ["keyword_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "landing_page_report": ReportStream( + report_definitions.LANDING_PAGE_REPORT_FIELDS, + ["landing_page_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "placeholder_feed_item_report": ReportStream( + report_definitions.PLACEHOLDER_FEED_ITEM_REPORT_FIELDS, + ["feed_item"], + resource_schema, + ["_sdc_record_hash"], + ), + "placeholder_report": ReportStream( + report_definitions.PLACEHOLDER_REPORT_FIELDS, + ["feed_placeholder_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "placement_performance_report": ReportStream( + report_definitions.PLACEMENT_PERFORMANCE_REPORT_FIELDS, + ["managed_placement_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "search_query_performance_report": ReportStream( + report_definitions.SEARCH_QUERY_PERFORMANCE_REPORT_FIELDS, + ["search_term_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "shopping_performance_report": ReportStream( + report_definitions.SHOPPING_PERFORMANCE_REPORT_FIELDS, + ["shopping_performance_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "user_location_performance_report": ReportStream( + report_definitions.USER_LOCATION_PERFORMANCE_REPORT_FIELDS, + ["user_location_view"], + resource_schema, + ["_sdc_record_hash"], + ), + "video_performance_report": ReportStream( + report_definitions.VIDEO_PERFORMANCE_REPORT_FIELDS, + ["video"], + resource_schema, + ["_sdc_record_hash"], + ), + } diff --git a/tap_google_ads/sync.py b/tap_google_ads/sync.py new file mode 100644 index 0000000..1628ad8 --- /dev/null +++ b/tap_google_ads/sync.py @@ -0,0 +1,45 @@ +import json + +import singer + +from tap_google_ads.client import create_sdk_client +from tap_google_ads.streams import initialize_core_streams, initialize_reports + +LOGGER = singer.get_logger() + + +def do_sync(config, catalog, resource_schema, state): + # QA ADDED WORKAROUND [START] + try: + customers = json.loads(config["login_customer_ids"]) + except TypeError: # falling back to raw value + customers = config["login_customer_ids"] + # QA ADDED WORKAROUND [END] + + selected_streams = [ + stream + for stream in catalog["streams"] + if singer.metadata.to_map(stream["metadata"])[()].get("selected") + ] + + core_streams = initialize_core_streams(resource_schema) + report_streams = initialize_reports(resource_schema) + + for customer in customers: + LOGGER.info(f"Syncing customer Id {customer['customerId']} ...") + sdk_client = create_sdk_client(config, customer["loginCustomerId"]) + for catalog_entry in selected_streams: + stream_name = catalog_entry["stream"] + mdata_map = singer.metadata.to_map(catalog_entry["metadata"]) + + primary_key = mdata_map[()].get("table-key-properties", []) + singer.messages.write_schema(stream_name, catalog_entry["schema"], primary_key) + + LOGGER.info(f"Syncing {stream_name} for customer Id {customer['customerId']}.") + + if core_streams.get(stream_name): + stream_obj = core_streams[stream_name] + else: + stream_obj = report_streams[stream_name] + + stream_obj.sync(sdk_client, customer, catalog_entry, config, state) diff --git a/tests/base.py b/tests/base.py index 59708bd..1990f6d 100644 --- a/tests/base.py +++ b/tests/base.py @@ -44,21 +44,13 @@ def get_type(): def get_properties(self, original: bool = True): """Configurable properties, with a switch to override the 'start_date' property""" return_value = { - 'start_date': '2020-12-01T00:00:00Z', - 'user_id': 'not used?', + 'start_date': '2021-12-01T00:00:00Z', + 'user_id': 'not used?', # TODO ? 'customer_ids': '5548074409,2728292456', - 'login_customer_ids': [ - { - "customerId": "5548074409", - "loginCustomerId": "2728292456", - }, - { - "customerId": "2728292456", - "loginCustomerId": "2728292456", - }, - ], + # 'conversion_window_days': '30', + 'login_customer_ids': [{"customerId": "5548074409", "loginCustomerId": "2728292456",}], } - + # TODO_TDL-17911 Add a test around conversion_window_days if original: return return_value @@ -73,7 +65,9 @@ def get_credentials(self): def expected_metadata(self): """The expected streams and metadata about the streams""" - + # TODO Investigate the foreign key expectations here, + # - must prove each uncommented entry is a true foregin key constraint. + # - must prove each commented entry is a NOT true foregin key constraint. return { # Core Objects "accounts": { @@ -85,9 +79,9 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.FULL_TABLE, self.FOREIGN_KEYS: { - 'accessible_bidding_strategy_id', - 'bidding_strategy_id', - 'campaign_budget_id', + # 'accessible_bidding_strategy_id', + # 'bidding_strategy_id', + # 'campaign_budget_id', 'customer_id' }, }, @@ -95,8 +89,8 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.FULL_TABLE, self.FOREIGN_KEYS: { - 'accessible_bidding_strategy_id', - 'bidding_strategy_id', + # 'accessible_bidding_strategy_id', + # 'bidding_strategy_id', 'campaign_id', 'customer_id', }, @@ -113,7 +107,10 @@ def expected_metadata(self): 'campaign_budgets': { self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.FULL_TABLE, - self.FOREIGN_KEYS: {"customer_id"}, + self.FOREIGN_KEYS: { + "customer_id", + "campaign_id", + }, }, 'bidding_strategies': { self.PRIMARY_KEYS:{"id"}, @@ -127,115 +124,123 @@ def expected_metadata(self): }, # Report objects "age_range_performance_report": { # "age_range_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "audience_performance_report": { # "campaign_audience_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "campaign_performance_report": { # "campaign_audience_view" - self.PRIMARY_KEYS: {"TODO"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - self.REPLICATION_KEYS: {"date"}, - }, - "call_metrics_call_details_report": { # "call_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, + # TODO Post Alpha + # "call_metrics_call_details_report": { # "call_view" + # self.PRIMARY_KEYS: {"_sdc_record_hash"}, + # self.REPLICATION_METHOD: self.INCREMENTAL, + # self.REPLICATION_KEYS: {"date"}, + # }, "click_performance_report": { # "click_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "display_keyword_performance_report": { # "display_keyword_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "display_topics_performance_report": { # "topic_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "gender_performance_report": { # "gender_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {"date"}, + }, + "geo_performance_report": { # "geographic_view" + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, - "geo_performance_report": { # "geographic_view", "user_location_view" - self.PRIMARY_KEYS: {"TODO"}, + "user_location_performance_report": { # "user_location_view" + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "keywordless_query_report": { # "dynamic_search_ads_search_term_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "keywords_performance_report": { # "keyword_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, - # TODO Do the land page reports have a different name in UI from the resource? - # TODO should they follow the _report naming convention "landing_page_report": { - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "expanded_landing_page_report": { - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "placeholder_feed_item_report": { # "feed_item", "feed_item_target" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "placeholder_report": { # "feed_placeholder_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "placement_performance_report": { # "managed_placement_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "search_query_performance_report": { # "search_term_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "shopping_performance_report": { # "shopping_performance_view" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {"date"}, + }, + "user_location_performance_report": { # "user_location_view" + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "video_performance_report": { # "video" - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, - # MISSING V1 reports "account_performance_report": { # accounts - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "adgroup_performance_report": { # ad_group - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, "ad_performance_report": { # ads - self.PRIMARY_KEYS: {"TODO"}, + self.PRIMARY_KEYS: {"_sdc_record_hash"}, self.REPLICATION_METHOD: self.INCREMENTAL, self.REPLICATION_KEYS: {"date"}, }, @@ -297,7 +302,8 @@ def expected_replication_keys(self): def expected_automatic_fields(self): auto_fields = {} for k, v in self.expected_metadata().items(): - auto_fields[k] = v.get(self.PRIMARY_KEYS, set()) | v.get(self.REPLICATION_KEYS, set()) + auto_fields[k] = v.get(self.PRIMARY_KEYS, set()) | v.get(self.REPLICATION_KEYS, set()) | \ + v.get(self.FOREIGN_KEYS, set()) return auto_fields @@ -449,8 +455,15 @@ def select_all_streams_and_fields(conn_id, catalogs, select_all_fields: bool = T connections.select_catalog_and_fields_via_metadata( conn_id, catalog, schema, [], non_selected_properties) + @staticmethod + def deselect_streams(conn_id, catalogs): + """Select all streams and all fields within streams""" + for catalog in catalogs: + schema = menagerie.get_annotated_schema(conn_id, catalog['stream_id']) - def _select_streams_and_fields(self, conn_id, catalogs, select_default_fields, select_pagination_fields): + connections.deselect_catalog_via_metadata(conn_id, catalog, schema) + + def _select_streams_and_fields(self, conn_id, catalogs, select_default_fields): """Select all streams and all fields within streams""" for catalog in catalogs: @@ -466,10 +479,6 @@ def _select_streams_and_fields(self, conn_id, catalogs, select_default_fields, s non_selected_properties = properties.difference( self.expected_default_fields()[catalog['stream_name']] ) - elif select_pagination_fields: - non_selected_properties = properties.difference( - self.expected_pagination_fields()[catalog['stream_name']] - ) else: non_selected_properties = properties @@ -519,9 +528,201 @@ def timedelta_formatted(self, dtime, days=0): ### Tap Specific Methods ########################################################################## + def select_all_streams_and_default_fields(self, conn_id, catalogs): + """Select all streams and all fields within streams""" + for catalog in catalogs: + if not self.is_report(catalog['tap_stream_id']): + raise RuntimeError("Method intended for report streams only.") + + schema_and_metadata = menagerie.get_annotated_schema(conn_id, catalog['stream_id']) + metadata = schema_and_metadata['metadata'] + properties = {md['breadcrumb'][-1] + for md in metadata + if len(md['breadcrumb']) > 0 and md['breadcrumb'][0] == 'properties'} + expected_fields = self.expected_default_fields()[catalog['stream_name']] + self.assertTrue(expected_fields.issubset(properties), + msg=f"{catalog['stream_name']} missing {expected_fields.difference(properties)}") + non_selected_properties = properties.difference(expected_fields) + connections.select_catalog_and_fields_via_metadata( + conn_id, catalog, schema_and_metadata, [], non_selected_properties + ) + def is_report(self, stream): return stream.endswith('_report') # TODO exclusion rules - # TODO core objects vs reports + @staticmethod + def expected_default_fields(): + """ + Report streams will select fields based on the default values that + are provided when selecting the report type in Google's UI when possible. + These fields do not translate perfectly to our report syncs and so a subset + of those fields are used in almost all cases here. + + returns a dictionary of reports to standard fields + """ + return { + 'ad_performance_report': { + 'average_cpc', # 'Avg. CPC', + 'clicks', # 'Clicks', + 'conversions', # 'Conversions', + 'cost_per_conversion', # 'Cost / conv.', + 'ctr', # 'CTR', + 'customer_id', # 'Customer ID', + 'impressions', # 'Impr.', + 'view_through_conversions', # 'View-through conv.', + }, + "adgroup_performance_report": { + 'average_cpc', # Avg. CPC, + 'clicks', # Clicks, + 'conversions', # Conversions, + 'cost_per_conversion', # Cost / conv., + 'ctr', # CTR, + 'customer_id', # Customer ID, + 'impressions', # Impr., + 'view_through_conversions', # View-through conv., + }, + "audience_performance_report": { + 'average_cpc', # Avg. CPC, + 'average_cpm', # Avg. CPM + 'clicks', # Clicks, + 'ctr', # CTR, + 'customer_id', # Customer ID, + 'impressions', # Impr., + 'ad_group_targeting_setting', # Targeting Setting, + }, + "campaign_performance_report": { + 'average_cpc', # Avg. CPC, + 'clicks', # Clicks, + 'conversions', # Conversions, + 'cost_per_conversion', # Cost / conv., + 'ctr', # CTR, + 'customer_id', # Customer ID, + 'impressions', # Impr., + 'view_through_conversions', # View-through conv., + }, + "click_performance_report": { + 'ad_group_ad', + 'ad_group_id', + 'ad_group_name', + 'ad_group_status', + 'ad_network_type', + 'area_of_interest', + 'campaign_location_target', + 'click_type', + 'clicks', + 'customer_descriptive_name', + 'customer_id', + 'device', + 'gclid', + 'location_of_presence', + 'month_of_year', + 'page_number', + 'slot', + 'user_list', + }, + "display_keyword_performance_report": { # TODO NO DATA AVAILABLE + 'average_cpc', # Avg. CPC, + 'average_cpm', # Avg. CPM, + 'average_cpv', # Avg. CPV, + 'clicks', # Clicks, + 'conversions', # Conversions, + 'cost_per_conversion', # Cost / conv., + 'impressions', # Impr., + 'interaction_rate', # Interaction rate, + 'interactions', # Interactions, + 'view_through_conversions', # View-through conv., + }, + "display_topics_performance_report": { # TODO NO DATA AVAILABLE + 'ad_group_name', # 'ad_group', # Ad group, + 'average_cpc', # Avg. CPC, + 'average_cpm', # Avg. CPM, + 'campaign_name', # 'campaign', # Campaign, + 'clicks', # Clicks, + 'ctr', # CTR, + 'customer_currency_code', # 'currency_code', # Currency code, + 'impressions', # Impr., + }, + "placement_performance_report": { # TODO NO DATA AVAILABLE + 'clicks', + 'impressions', # Impr., + 'ad_group_criterion_placement', # 'placement_group', 'placement_type', + }, + # "keywords_performance_report": set(), + # "shopping_performance_report": set(), + "video_performance_report": { + 'campaign_name', + 'clicks', + 'video_quartile_p25_rate', + }, + # NOTE AFTER THIS POINT COULDN"T FIND IN UI + "account_performance_report": { + 'average_cpc', + 'click_type', + 'clicks', + 'date', + 'descriptive_name', + 'id', + 'impressions', + 'invalid_clicks', + 'manager', + 'test_account', + 'time_zone', + }, + "geo_performance_report": { + 'clicks', + 'ctr', # CTR, + 'impressions', # Impr., + 'average_cpc', + 'conversions', + 'view_through_conversions', # View-through conv., + 'cost_per_conversion', # Cost / conv., + 'geo_target_region', + }, + "gender_performance_report": { + 'ad_group_criterion_gender', + 'average_cpc', # Avg. CPC, + 'clicks', # Clicks, + 'conversions', # Conversions, + 'cost_per_conversion', # Cost / conv., + 'ctr', # CTR, + 'customer_id', # Customer ID, + 'impressions', # Impr., + 'view_through_conversions', # View-through conv., + }, + "search_query_performance_report": { + 'clicks', + 'ctr', # CTR, + 'impressions', # Impr., + 'average_cpc', + 'conversions', + 'view_through_conversions', # View-through conv., + 'cost_per_conversion', # Cost / conv., + 'search_term', + 'search_term_match_type', + }, + "age_range_performance_report": { + 'clicks', + 'ctr', # CTR, + 'impressions', # Impr., + 'average_cpc', + 'conversions', + 'view_through_conversions', # View-through conv., + 'cost_per_conversion', # Cost / conv., + 'ad_group_criterion_age_range', # 'Age', + }, + 'placeholder_feed_item_report': { + 'clicks', + 'impressions', + 'placeholder_type', + }, + 'placeholder_report': { + 'clicks', + 'cost_micros', + 'interactions', + 'placeholder_type', + }, + # 'landing_page_report': set(), # TODO + # 'expanded_landing_page_report': set(), # TODO + } diff --git a/tests/test_google_ads_automatic_fields.py b/tests/test_google_ads_automatic_fields.py new file mode 100644 index 0000000..4491697 --- /dev/null +++ b/tests/test_google_ads_automatic_fields.py @@ -0,0 +1,121 @@ +"""Test tap discovery mode and metadata.""" +import re + +from tap_tester import menagerie, connections, runner + +from base import GoogleAdsBase + + +class AutomaticFieldsGoogleAds(GoogleAdsBase): + """ + Test tap's sync mode can extract records for all streams + with minimum field selection. + """ + + @staticmethod + def name(): + return "tt_google_ads_auto_fields" + + def test_error_case(self): + """ + Testing that basic sync with minimum field selection results in Critical Errors with clear message. + """ + print("Automatic Fields Test for tap-google-ads report streams") + + # --- Test report streams throw an error --- # + + streams_to_test = {stream for stream in self.expected_streams() + if self.is_report(stream)} + + conn_id = connections.ensure_connection(self) + + # Run a discovery job + found_catalogs = self.run_and_verify_check_mode(conn_id) + + for stream in streams_to_test: + with self.subTest(stream=stream): + + catalogs_to_test = [catalog + for catalog in found_catalogs + if catalog["stream_name"] == stream] + + # select all fields for core streams and... + self.select_all_streams_and_fields( + conn_id, + catalogs_to_test, + select_all_fields=False + ) + try: + # Run a sync + sync_job_name = runner.run_sync_mode(self, conn_id) + + exit_status = menagerie.get_exit_status(conn_id, sync_job_name) + + self.assertEqual(1, exit_status.get('tap_exit_status')) + self.assertEqual(0, exit_status.get('target_exit_status')) + self.assertEqual(0, exit_status.get('discovery_exit_status')) + self.assertIsNone(exit_status.get('check_exit_status')) + + # Verify error message tells user they must select an attribute/metric for the invalid stream + self.assertIn( + "Please select at least one attribute and metric in order to replicate", + exit_status.get("tap_error_message") + ) + self.assertIn(stream, exit_status.get("tap_error_message")) + + finally: + # deselect stream once it's been tested + self.deselect_streams(conn_id, catalogs_to_test) + + def test_happy_path(self): + """ + Testing that basic sync with minimum field selection functions without Critical Errors + """ + print("Automatic Fields Test for tap-google-ads core streams") + + # --- Start testing core streams --- # + + conn_id = connections.ensure_connection(self) + + streams_to_test = {stream for stream in self.expected_streams() + if not self.is_report(stream)} + + # Run a discovery job + found_catalogs = self.run_and_verify_check_mode(conn_id) + + # Perform table and field selection... + catalogs_to_test = [catalog for catalog in found_catalogs + if catalog['stream_name'] in streams_to_test] + # select all fields for core streams and... + self.select_all_streams_and_fields(conn_id, catalogs_to_test, select_all_fields=False) + + # Run a sync + sync_job_name = runner.run_sync_mode(self, conn_id) + + # Verify the tap and target do not throw a critical error + exit_status = menagerie.get_exit_status(conn_id, sync_job_name) + menagerie.verify_sync_exit_status(self, exit_status, sync_job_name) + + # acquire records from target output + synced_records = runner.get_records_from_target_output() + + for stream in streams_to_test: + with self.subTest(stream=stream): + + # # Verify that only the automatic fields are sent to the target. + expected_auto_fields = self.expected_automatic_fields() + expected_primary_key = list(self.expected_primary_keys()[stream])[0] # assumes no compound-pks + self.assertEqual(len(self.expected_primary_keys()[stream]), 1, msg="Compound pk not supported") + for record in synced_records[stream]['messages']: + + record_primary_key_values = record['data'][expected_primary_key] + record_keys = set(record['data'].keys()) + + with self.subTest(primary_key=record_primary_key_values): + self.assertSetEqual(expected_auto_fields[stream], record_keys) + + # Verify that all replicated records have unique primary key values. + actual_pks = [row.get('data').get(expected_primary_key) for row in + synced_records.get(stream, {'messages':[]}).get('messages', []) if row.get('data')] + + self.assertCountEqual(actual_pks, set(actual_pks)) diff --git a/tests/test_google_ads_bookmarks.py b/tests/test_google_ads_bookmarks.py index 3d3b095..33244b9 100644 --- a/tests/test_google_ads_bookmarks.py +++ b/tests/test_google_ads_bookmarks.py @@ -1,13 +1,15 @@ -"""Test tap discovery mode and metadata.""" +"""Test tap bookmarks and converstion window.""" import re +from datetime import datetime as dt +from datetime import timedelta from tap_tester import menagerie, connections, runner from base import GoogleAdsBase -class DiscoveryTest(GoogleAdsBase): - """Test tap discovery mode and metadata conforms to standards.""" +class BookmarksTest(GoogleAdsBase): + """Test tap bookmarks.""" @staticmethod def name(): @@ -15,54 +17,45 @@ def name(): def test_run(self): """ - Testing that basic sync functions without Critical Errors + Testing that the tap sets and uses bookmarks correctly """ print("Bookmarks Test for tap-google-ads") conn_id = connections.ensure_connection(self) streams_to_test = self.expected_streams() - { - # TODO we are only testing core strems at the moment - 'landing_page_report', - 'expanded_landing_page_report', + 'audience_performance_report', + 'display_keyword_performance_report', 'display_topics_performance_report', - 'call_metrics_call_details_report', - 'gender_performance_report', - 'search_query_performance_report', - 'placeholder_feed_item_report', + 'expanded_landing_page_report', + 'keywordless_query_report', 'keywords_performance_report', - 'video_performance_report', - 'campaign_performance_report', - 'geo_performance_report', - 'placeholder_report', + 'landing_page_report', 'placement_performance_report', - 'click_performance_report', - 'display_keyword_performance_report', + 'search_query_performance_report', 'shopping_performance_report', - 'ad_performance_report', - 'age_range_performance_report', - 'keywordless_query_report', - 'account_performance_report', - 'adgroup_performance_report', - 'audience_performance_report', + 'user_location_performance_report', + 'video_performance_report', } # Run a discovery job - check_job_name = runner.run_check_mode(self, conn_id) - exit_status = menagerie.get_exit_status(conn_id, check_job_name) - menagerie.verify_check_exit_status(self, exit_status, check_job_name) + found_catalogs_1 = self.run_and_verify_check_mode(conn_id) - # Verify a catalog was produced for each stream under test - found_catalogs = menagerie.get_catalogs(conn_id) - self.assertGreater(len(found_catalogs), 0) - found_catalog_names = {found_catalog['stream_name'] for found_catalog in found_catalogs} - self.assertSetEqual(streams_to_test, found_catalog_names) + # partition catalogs for use in table/field seelction + test_catalogs_1 = [catalog for catalog in found_catalogs_1 + if catalog.get('stream_name') in streams_to_test] + core_catalogs_1 = [catalog for catalog in test_catalogs_1 + if not self.is_report(catalog['stream_name'])] + report_catalogs_1 = [catalog for catalog in test_catalogs_1 + if self.is_report(catalog['stream_name'])] - # Perform table and field selection - self.select_all_streams_and_fields(conn_id, found_catalogs, select_all_fields=True) + # select all fields for core streams + self.select_all_streams_and_fields(conn_id, core_catalogs_1, select_all_fields=True) + # select 'default' fields for report streams + self.select_all_streams_and_default_fields(conn_id, report_catalogs_1) - # Run a sync + # Run a sync sync_job_name_1 = runner.run_sync_mode(self, conn_id) # Verify the tap and target do not throw a critical error @@ -72,6 +65,26 @@ def test_run(self): # acquire records from target output synced_records_1 = runner.get_records_from_target_output() state_1 = menagerie.get_state(conn_id) + bookmarks_1 = state_1.get('bookmarks') + currently_syncing_1 = state_1.get('currently_syncing', 'KEY NOT SAVED IN STATE') + + # TODO_TDL-17918 Determine if we can test all cases at the tap-tester level + # TEST CASE 1: state > today - converstion window, time format 2. Will age out and become TC2 Feb 24, 22 + # TEST CASE 2: state < today - converstion window, time format 1 + manipulated_state = {'currently_syncing': 'None', + 'bookmarks': { + 'adgroup_performance_report': {'date': '2022-01-24T00:00:00.000000Z'}, + 'geo_performance_report': {'date': '2022-01-24T00:00:00.000000Z'}, + 'gender_performance_report': {'date': '2022-01-24T00:00:00.000000Z'}, + 'placeholder_feed_item_report': {'date': '2021-12-30T00:00:00.000000Z'}, + 'age_range_performance_report': {'date': '2022-01-24T00:00:00.000000Z'}, + 'account_performance_report': {'date': '2022-01-24T00:00:00.000000Z'}, + 'click_performance_report': {'date': '2022-01-24T00:00:00.000000Z'}, + 'campaign_performance_report': {'date': '2022-01-24T00:00:00.000000Z'}, + 'placeholder_report': {'date': '2021-12-30T00:00:00.000000Z'}, + 'ad_performance_report': {'date': '2022-01-24T00:00:00.000000Z'}, + }} + menagerie.set_state(conn_id, manipulated_state) # Run another sync sync_job_name_2 = runner.run_sync_mode(self, conn_id) @@ -83,36 +96,115 @@ def test_run(self): # acquire records from target output synced_records_2 = runner.get_records_from_target_output() state_2 = menagerie.get_state(conn_id) - - + bookmarks_2 = state_2.get('bookmarks') + currently_syncing_2 = state_2.get('currently_syncing', 'KEY NOT SAVED IN STATE') + + # Checking syncs were successful prior to stream-level assertions + with self.subTest(): + + # Verify sync is not interrupted by checking currently_syncing in state for sync 1 + self.assertIsNone(currently_syncing_1) + # Verify bookmarks are saved + self.assertIsNotNone(bookmarks_1) + + # Verify sync is not interrupted by checking currently_syncing in state for sync 2 + self.assertIsNone(currently_syncing_2) + # Verify bookmarks are saved + self.assertIsNotNone(bookmarks_2) + + # Verify ONLY report streams under test have bookmark entries in state + expected_incremental_streams = {stream for stream in streams_to_test if self.is_report(stream)} + unexpected_incremental_streams_1 = {stream for stream in bookmarks_1.keys() if stream not in expected_incremental_streams} + unexpected_incremental_streams_2 = {stream for stream in bookmarks_2.keys() if stream not in expected_incremental_streams} + self.assertSetEqual(set(), unexpected_incremental_streams_1) + self.assertSetEqual(set(), unexpected_incremental_streams_2) + + # stream-level assertions for stream in streams_to_test: with self.subTest(stream=stream): # set expectations expected_replication_method = self.expected_replication_method()[stream] + conversion_window = timedelta(days=30) # defaulted value # gather results records_1 = [message['data'] for message in synced_records_1[stream]['messages']] records_2 = [message['data'] for message in synced_records_2[stream]['messages']] record_count_1 = len(records_1) record_count_2 = len(records_2) - bookmarks_1 = state_1.get(stream) - bookmarks_2 = state_2.get(stream) - - # sanity check WIP - print(f"Stream: {stream} \n" - f"Record 1 Sync 1: {records_1[0]}") - # end WIP + stream_bookmark_1 = bookmarks_1.get(stream) + stream_bookmark_2 = bookmarks_2.get(stream) if expected_replication_method == self.INCREMENTAL: # included to catch a contradiction in our base expectations - if not stream.endswith('_report'): + if not self.is_report(stream): raise AssertionError( f"Only Reports streams should be expected to support {expected_replication_method} replication." ) - # TODO need to finish implementing test cases for report streams + expected_replication_key = list(self.expected_replication_keys()[stream])[0] # assumes 1 value + manipulated_bookmark = manipulated_state['bookmarks'][stream] + today_minus_conversion_window = dt.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) - conversion_window + today = dt.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) + manipulated_state_formatted = dt.strptime(manipulated_bookmark.get(expected_replication_key), self.REPLICATION_KEY_FORMAT) + if manipulated_state_formatted < today_minus_conversion_window: + reference_time = manipulated_state_formatted + else: + reference_time = today_minus_conversion_window + + # Verify bookmarks saved match formatting standards for sync 1 + self.assertIsNotNone(stream_bookmark_1) + bookmark_value_1 = stream_bookmark_1.get(expected_replication_key) + self.assertIsNotNone(bookmark_value_1) + self.assertIsInstance(bookmark_value_1, str) + try: + parsed_bookmark_value_1 = dt.strptime(bookmark_value_1, self.REPLICATION_KEY_FORMAT) + except ValueError as err: + raise AssertionError( + f"Bookmarked value does not conform to expected format: {self.REPLICATION_KEY_FORMAT}" + ) from err + + # Verify bookmarks saved match formatting standards for sync 2 + self.assertIsNotNone(stream_bookmark_2) + bookmark_value_2 = stream_bookmark_2.get(expected_replication_key) + self.assertIsNotNone(bookmark_value_2) + self.assertIsInstance(bookmark_value_2, str) + + try: + parsed_bookmark_value_2 = dt.strptime(bookmark_value_2, self.REPLICATION_KEY_FORMAT) + except ValueError as err: + + try: + parsed_bookmark_value_2 = dt.strptime(bookmark_value_2, "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError as err: + + raise AssertionError( + f"Bookmarked value does not conform to expected formats: " + + "\n Format 1: {}".format(self.REPLICATION_KEY_FORMAT) + + "\n Format 2: %Y-%m-%dT%H:%M:%S.%fZ" + ) from err + + # Verify the bookmark is set based on sync execution time + self.assertGreaterEqual(parsed_bookmark_value_1, today) # TODO can we get more sepecifc with this? + self.assertGreaterEqual(parsed_bookmark_value_2, today) + + # Verify 2nd sync only replicates records newer than reference_time + for record in records_2: + rec_time = dt.strptime(record.get(expected_replication_key), self.REPLICATION_KEY_FORMAT) + self.assertGreaterEqual(rec_time, reference_time, \ + msg="record time cannot be less than reference time: {}".format(reference_time) + ) + + # Verify the number of records in records_1 where sync >= reference_time + # matches the number of records in records_2 + records_1_after_manipulated_bookmark = 0 + for record in records_1: + rec_time = dt.strptime(record.get(expected_replication_key), self.REPLICATION_KEY_FORMAT) + if rec_time >= reference_time: + records_1_after_manipulated_bookmark += 1 + self.assertEqual(records_1_after_manipulated_bookmark, record_count_2, \ + msg="Expected {} records in each sync".format(records_1_after_manipulated_bookmark)) elif expected_replication_method == self.FULL_TABLE: @@ -120,8 +212,8 @@ def test_run(self): self.assertEqual(record_count_1, record_count_2) # Verify full table streams do not save bookmarked values at the conclusion of a succesful sync - self.assertIsNone(bookmarks_1) - self.assertIsNone(bookmarks_2) + self.assertIsNone(stream_bookmark_1) + self.assertIsNone(stream_bookmark_2) # Verify full table streams replicate the same number of records on each sync self.assertEqual(record_count_1, record_count_2) @@ -129,9 +221,10 @@ def test_run(self): # Verify full tables streams replicate the exact same set of records on each sync for record in records_1: self.assertIn(record, records_2) - + # Verify at least 1 record was replicated for each stream self.assertGreater(record_count_1, 0) - - - print(f"{stream} {record_count_1} records replicated.") + self.assertGreater(record_count_2, 0) + + + print(f"{stream} sync 2 records replicated: {record_count_2}") diff --git a/tests/test_google_ads_discovery.py b/tests/test_google_ads_discovery.py index 3e8f531..e4ab434 100644 --- a/tests/test_google_ads_discovery.py +++ b/tests/test_google_ads_discovery.py @@ -10,14 +10,14 @@ class DiscoveryTest(GoogleAdsBase): """Test tap discovery mode and metadata conforms to standards.""" def expected_fields(self): - """The expected streams and metadata about the streams""" - # TODO verify accounts, ads, ad_groups, campaigns contain foreign keys for - # 'campaign_budgets', 'bidding_strategies', 'accessible_bidding_strategies' - # and only foreign keys BUT CHECK DOCS + """ + The expected streams and metadata about the streams. + TODO's in this method will be picked up as part of TDL-17909 + """ return { # Core Objects - "accounts": { # TODO check with Brian on changes + "accounts": { # TODO_TDL-17909 check with Brian on changes # OLD FIELDS (with mapping) "currency_code", "id", # "customer_id", @@ -41,7 +41,7 @@ def expected_fields(self): 'remarketing_setting.google_global_site_tag', 'tracking_url_template', }, - "campaigns": { # TODO check out nested keys once these are satisfied + "campaigns": { # TODO_TDL-17909 check out nested keys once these are satisfied # OLD FIELDS "ad_serving_optimization_status", "advertising_channel_type", @@ -124,12 +124,12 @@ def expected_fields(self): "vanity_pharma.vanity_pharma_text", "video_brand_safety_suitability", }, - "ad_groups": { # TODO check out nested keys once these are satisfied + "ad_groups": { # TODO_TDL-17909 check out nested keys once these are satisfied # OLD FIELDS (with mappings) "type", # ("ad_group_type") "base_ad_group", # ("base_ad_group_id") # "bidding_strategy_configuration", # DNE - "campaign", #("campaign_name", "campaign_id", "base_campaign_id") # TODO redo this + "campaign", #("campaign_name", "campaign_id", "base_campaign_id") # TODO_TDL-17909 redo this "id", "labels", "name", @@ -161,7 +161,7 @@ def expected_fields(self): "target_cpa_micros", "effective_target_roas", }, - "ads": { # TODO check out nested keys once these are satisfied + "ads": { # TODO_TDL-17909 check out nested keys once these are satisfied # OLD FIELDS (with mappings) "ad_group_id", "base_ad_group_id", @@ -199,10 +199,8 @@ def expected_fields(self): }, "display_keyword_performance_report": { # "display_keyword_view" }, - "display_topics_performance_report":{ # "topic_view" - }, - "": { # "topic_view" todo consult https://developers.google.com/google-ads/api/docs/migration/url-reports for migrating this report - }, + "display_topics_performance_report": { # "topic_view" + },# TODO_TDL-17909 consult https://developers.google.com/google-ads/api/docs/migration/url-reports for migrating this report "gender_performance_report": { # "gender_view" }, "geo_performance_report": { # "geographic_view", "user_location_view" @@ -233,7 +231,7 @@ def expected_fields(self): }, "ad_performance_report": { # ads }, - # Custom Reports TODO feature + # Custom Reports [OUTSIDE SCOPE OF ALPHA] } @staticmethod @@ -259,44 +257,11 @@ def test_run(self): """ print("Discovery Test for tap-google-ads") - conn_id = connections.ensure_connection(self) + streams_to_test = self.expected_streams() - streams_to_test = self.expected_streams() - { - # BUG_2 | missing - 'landing_page_report', - 'expanded_landing_page_report', - 'display_topics_performance_report', - 'call_metrics_call_details_report', - 'gender_performance_report', - 'search_query_performance_report', - 'placeholder_feed_item_report', - 'keywords_performance_report', - 'video_performance_report', - 'campaign_performance_report', - 'geo_performance_report', - 'placeholder_report', - 'placement_performance_report', - 'click_performance_report', - 'display_keyword_performance_report', - 'shopping_performance_report', - 'ad_performance_report', - 'age_range_performance_report', - 'keywordless_query_report', - 'account_performance_report', - 'adgroup_performance_report', - 'audience_performance_report', - } + conn_id = connections.ensure_connection(self) - # found_catalogs = self.run_and_verify_check_mode(conn_id) # TODO PUT BACK - # TODO REMOVE FROM HERE - check_job_name = runner.run_check_mode(self, conn_id) - exit_status = menagerie.get_exit_status(conn_id, check_job_name) - menagerie.verify_check_exit_status(self, exit_status, check_job_name) - found_catalogs = menagerie.get_catalogs(conn_id) - self.assertGreater(len(found_catalogs), 0) - found_catalog_names = {found_catalog['stream_name'] for found_catalog in found_catalogs} - self.assertSetEqual(streams_to_test, found_catalog_names) - # TODO TO HERE + found_catalogs = self.run_and_verify_check_mode(conn_id) # Verify stream names follow naming convention # streams should only have lowercase alphas and underscores @@ -304,7 +269,7 @@ def test_run(self): self.assertTrue(all([re.fullmatch(r"[a-z_]+", name) for name in found_catalog_names]), msg="One or more streams don't follow standard naming") - for stream in streams_to_test: # {'accounts', 'campaigns', 'ad_groups', 'ads'}: # # TODO PUT BACK + for stream in streams_to_test: with self.subTest(stream=stream): # Verify the catalog is found for a given stream @@ -312,15 +277,17 @@ def test_run(self): if catalog["stream_name"] == stream])) self.assertIsNotNone(catalog) - # collecting expected values + # collecting expected values from base.py expected_primary_keys = self.expected_primary_keys()[stream] expected_foreign_keys = self.expected_foreign_keys()[stream] expected_replication_keys = self.expected_replication_keys()[stream] - expected_automatic_fields = expected_primary_keys | expected_replication_keys + expected_automatic_fields = expected_primary_keys | expected_replication_keys | expected_foreign_keys expected_replication_method = self.expected_replication_method()[stream] - expected_fields = self.expected_fields()[stream] + # expected_fields = self.expected_fields()[stream] # TODO_TDL-17909 + is_report = self.is_report(stream) + expected_behaviors = {'METRIC', 'SEGMENT', 'ATTRIBUTE'} if is_report else {'ATTRIBUTE', 'SEGMENT'} - # collecting actual values + # collecting actual values from the catalog schema_and_metadata = menagerie.get_annotated_schema(conn_id, catalog['stream_id']) metadata = schema_and_metadata["metadata"] stream_properties = [item for item in metadata if item.get("breadcrumb") == []] @@ -328,10 +295,6 @@ def test_run(self): stream_properties[0].get( "metadata", {self.PRIMARY_KEYS: []}).get(self.PRIMARY_KEYS, []) ) - actual_foreign_keys = set( - stream_properties[0].get( - "metadata", {self.FOREIGN_KEYS: []}).get(self.FOREIGN_KEYS, []) - ) actual_replication_keys = set( stream_properties[0].get( "metadata", {self.REPLICATION_KEYS: []}).get(self.REPLICATION_KEYS, []) @@ -340,12 +303,17 @@ def test_run(self): "metadata", {self.REPLICATION_METHOD: None}).get(self.REPLICATION_METHOD) actual_automatic_fields = set( item.get("breadcrumb", ["properties", None])[1] for item in metadata - if item.get("metadata").get("inclusion") == "automatic" + if item.get("metadata").get("inclusion") == "automatic" ) actual_fields = [] for md_entry in metadata: if md_entry['breadcrumb'] != []: actual_fields.append(md_entry['breadcrumb'][1]) + fields_to_behaviors = {item['breadcrumb'][-1]: item['metadata']['behavior'] + for item in metadata + if item.get("breadcrumb", []) != [] + and item.get("metadata").get("behavior")} + fields_with_behavior = set(fields_to_behaviors.keys()) ########################################################################## ### metadata assertions @@ -357,55 +325,69 @@ def test_run(self): "\nstream_properties | {}".format(stream_properties)) # verify there are no duplicate metadata entries - #self.assertEqual(len(actual_fields), len(set(actual_fields)), msg = f"duplicates in the fields retrieved") + self.assertEqual(len(actual_fields), len(set(actual_fields)), msg="duplicates in the fields retrieved") + - # TODO BUG (unclear on significance in saas tap ?) # verify the tap_stream_id and stream_name are consistent (only applies to SaaS taps) - # self.assertEqual(stream_properties[0]['stream_name'], stream_properties[0]['tap_stream_id']) + self.assertEqual(catalog['stream_name'], catalog['tap_stream_id']) - # BUG_TDL_17533 - # [tap-google-ads] Primary keys have incorrect name for core objects # verify primary key(s) - # self.assertSetEqual(expected_primary_keys, actual_primary_keys) # BUG_TDL_17533 + self.assertSetEqual(expected_primary_keys, actual_primary_keys) - # BUG_1' | all core streams are missing this metadata TODO does this thing even get used ANYWHERE? # verify replication method - # self.assertEqual(expected_replication_method, actual_replication_method) - - # verify replication key(s) - self.assertSetEqual(expected_replication_keys, actual_replication_keys) - - # TODO | implement when foreign keys are complete - # verify foreign keys are present for each core stream - # self.assertSetEqual(expected_foreign_keys, actual_foreign_keys) + self.assertEqual(expected_replication_method, actual_replication_method) - # verify foreign keys are given inclusion of automatic - - # verify replication key is present for any stream with replication method = INCREMENTAL + # verify replication key is present for any stream with replication method is INCREMENTAL if actual_replication_method == 'INCREMENTAL': - # TODO | Implement at time sync is working - # self.assertEqual(expected_replication_keys, actual_replication_keys) - pass + self.assertNotEqual(actual_replication_keys, set()) else: self.assertEqual(actual_replication_keys, set()) - # verify all expected fields are found # TODO set expectations + # verify replication key(s) + self.assertSetEqual(expected_replication_keys, actual_replication_keys) + + # verify all expected fields are found # TODO_TDL-17909 set expectations # self.assertSetEqual(expected_fields, set(actual_fields)) # verify the stream is given the inclusion of available self.assertEqual(catalog['metadata']['inclusion'], 'available', msg=f"{stream} cannot be selected") - # verify the primary, replication keys are given the inclusions of automatic - #self.assertSetEqual(expected_automatic_fields, actual_automatic_fields) + # verify the primary, replication keys and foreign keys are given the inclusions of automatic + self.assertSetEqual(expected_automatic_fields, actual_automatic_fields) # verify all other fields are given inclusion of available self.assertTrue( - all({item.get("metadata").get("inclusion") in {"available", "unsupported"} + all({item.get("metadata").get("inclusion") in {"available"} for item in metadata if item.get("breadcrumb", []) != [] and item.get("breadcrumb", ["properties", None])[1] not in actual_automatic_fields}), msg="Not all non key properties are set to available in metadata") - # verify field exclusions for each strema match our expectations - # TODO further tests may be needed, including attempted syncs with invalid field combos + # verify 'behavior' is present in metadata for all streams + if is_report: + actual_fields.remove('_sdc_record_hash') + self.assertEqual(fields_with_behavior, set(actual_fields)) + + # verify 'behavior' falls into expected set of behaviors (based on stream type) + for field, behavior in fields_to_behaviors.items(): + with self.subTest(field=field): + self.assertIn(behavior, expected_behaviors) + + # NB | The following assertion is left commented with the assumption that this will be a valid + # expectation by the time the tap moves to Beta. If this is not valid at that time it should + # be removed. Or if work done in TDL-17910 results in this being an unnecessary check + + # if is_report: + # # verify each field in a report stream has a 'fieldExclusions' entry and that the fields listed + # # in that set are present in elsewhere in the stream's catalog + # fields_to_exclusions = {md['breadcrumb'][-1]: md['metadata']['fieldExclusions'] + # for md in metadata + # if md['breadcrumb'] != [] and + # md['metadata'].get('fieldExclusions')} + # for field, exclusions in fields_to_exclusions.items(): + # with self.subTest(field=field): + # self.assertTrue( + # set(exclusions).issubset(set(actual_fields)), + # msg=f"'fieldExclusions' contain fields not accounted for by the catalog: {set(exclusions) - set(actual_fields)}" + # ) diff --git a/tests/test_google_ads_start_date.py b/tests/test_google_ads_start_date.py index 7025df7..10d6455 100644 --- a/tests/test_google_ads_start_date.py +++ b/tests/test_google_ads_start_date.py @@ -1,52 +1,24 @@ import os +from datetime import datetime as dt + from tap_tester import connections, runner, menagerie from base import GoogleAdsBase class StartDateTest(GoogleAdsBase): + """A shared test to be ran with various configurations of start dates and streams to test""" - start_date_1 = "" - start_date_2 = "" - - @staticmethod - def name(): - return "tt_google_ads_start_date" - - def test_run(self): + def run_test(self): """Instantiate start date according to the desired data set and run the test""" - self.start_date_1 = self.get_properties().get('start_date') # '2020-12-01T00:00:00Z', - self.start_date_2 = self.timedelta_formatted(self.start_date_1, days=15) - self.start_date = self.start_date_1 - streams_to_test = self.expected_streams() - { - # TODO we are only testing core strems at the moment - 'landing_page_report', - 'expanded_landing_page_report', - 'display_topics_performance_report', - 'call_metrics_call_details_report', - 'gender_performance_report', - 'search_query_performance_report', - 'placeholder_feed_item_report', - 'keywords_performance_report', - 'video_performance_report', - 'campaign_performance_report', - 'geo_performance_report', - 'placeholder_report', - 'placement_performance_report', - 'click_performance_report', - 'display_keyword_performance_report', - 'shopping_performance_report', - 'ad_performance_report', - 'age_range_performance_report', - 'keywordless_query_report', - 'account_performance_report', - 'adgroup_performance_report', - 'audience_performance_report', - } + streams_to_test = self.streams_to_test + print(f"Streams under test {streams_to_test}") + print(f"Start Date 1: {self.start_date_1}") + print(f"Start Date 2: {self.start_date_2}") ########################################################################## ### Sync with Connection 1 @@ -56,27 +28,20 @@ def test_run(self): conn_id_1 = connections.ensure_connection(self) # run check mode - check_job_name_1 = runner.run_check_mode(self, conn_id_1) # TODO REMOVE START - exit_status_1 = menagerie.get_exit_status(conn_id_1, check_job_name_1) - menagerie.verify_check_exit_status(self, exit_status_1, check_job_name_1) - found_catalogs_1 = menagerie.get_catalogs(conn_id_1) # TODO REMOVE END - # found_catalogs_1 = self.run_and_verify_check_mode(conn_id_1) # TODO PUT BACK + found_catalogs_1 = self.run_and_verify_check_mode(conn_id_1) # table and field selection test_catalogs_1 = [catalog for catalog in found_catalogs_1 if catalog.get('stream_name') in streams_to_test] - self.select_all_streams_and_fields(conn_id_1, test_catalogs_1, select_all_fields=True) # TODO REMOVE - # self.perform_and_verify_table_and_field_selection(conn_id_1, test_catalogs_1, select_all_fields=True) # TODO PUT BACK + core_catalogs_1 = [catalog for catalog in test_catalogs_1 if not self.is_report(catalog['stream_name'])] + report_catalogs_1 = [catalog for catalog in test_catalogs_1 if self.is_report(catalog['stream_name'])] + # select all fields for core streams and... + self.select_all_streams_and_fields(conn_id_1, core_catalogs_1, select_all_fields=True) + # select 'default' fields for report streams + self.select_all_streams_and_default_fields(conn_id_1, report_catalogs_1) # run initial sync - sync_job_name_1 = runner.run_sync_mode(self, conn_id_1) # TODO REMOVE START - exit_status_1 = menagerie.get_exit_status(conn_id_1, sync_job_name_1) - menagerie.verify_sync_exit_status(self, exit_status_1, sync_job_name_1) - # BUG_TDL-17535 [tap-google-ads] Primary keys do not persist to target - # record_count_by_stream_1 = runner.examine_target_output_file( - # self, conn_id_1, streams_to_test, self.expected_primary_keys()) # BUG_TDL-17535 - # TODO REMOVE END - # record_count_by_stream_1 = self.run_and_verify_sync(conn_id_1) # TODO PUT BACK + record_count_by_stream_1 = self.run_and_verify_sync(conn_id_1) synced_records_1 = runner.get_records_from_target_output() ########################################################################## @@ -94,96 +59,118 @@ def test_run(self): conn_id_2 = connections.ensure_connection(self, original_properties=False) # run check mode - check_job_name_2 = runner.run_check_mode(self, conn_id_2) # TODO REMOVE START - exit_status_2 = menagerie.get_exit_status(conn_id_2, check_job_name_2) - menagerie.verify_check_exit_status(self, exit_status_2, check_job_name_2) - found_catalogs_2 = menagerie.get_catalogs(conn_id_2) # TODO REMOVE END - # found_catalogs_2 = self.run_and_verify_check_mode(conn_id_2) # TODO PUT BACK - + found_catalogs_2 = self.run_and_verify_check_mode(conn_id_2) # table and field selection test_catalogs_2 = [catalog for catalog in found_catalogs_2 if catalog.get('stream_name') in streams_to_test] - self.select_all_streams_and_fields(conn_id_2, test_catalogs_2, select_all_fields=True) # TODO REMOVE - # self.perform_and_verify_table_and_field_selection(conn_id_2, test_catalogs_2, select_all_fields=True) # TODO PUT BACK - + core_catalogs_2 = [catalog for catalog in test_catalogs_2 if not self.is_report(catalog['stream_name'])] + report_catalogs_2 = [catalog for catalog in test_catalogs_2 if self.is_report(catalog['stream_name'])] + # select all fields for core streams and... + self.select_all_streams_and_fields(conn_id_2, core_catalogs_2, select_all_fields=True) + # select 'default' fields for report streams + self.select_all_streams_and_default_fields(conn_id_2, report_catalogs_2) # run sync - sync_job_name_2 = runner.run_sync_mode(self, conn_id_2) # TODO REMOVE START - exit_status_2 = menagerie.get_exit_status(conn_id_2, sync_job_name_2) - menagerie.verify_sync_exit_status(self, exit_status_2, sync_job_name_2) - # BUG_TDL-17535 [tap-google-ads] Primary keys do not persist to target - # record_count_by_stream_2 = runner.examine_target_output_file( - # self, conn_id_2, streams_to_test, self.expected_primary_keys()) # BUG_TDL-17535 - # TODO REMOVE END - # record_count_by_stream_2 = self.run_and_verify_sync(conn_id_2) # TODO PUT BACK + record_count_by_stream_2 = self.run_and_verify_sync(conn_id_2) synced_records_2 = runner.get_records_from_target_output() for stream in streams_to_test: with self.subTest(stream=stream): # expected values - # expected_primary_keys = self.expected_primary_keys()[stream] # BUG_TDL_17533 - expected_primary_keys = {f'{stream}.id'} # BUG_TDL_17533 - - # TODO update this with the lookback window DOES IT APPLY TO START DATE? - # expected_conversion_window = -1 * int(self.get_properties()['conversion_window']) - expected_start_date_1 = self.timedelta_formatted(self.start_date_1, days=0) # expected_conversion_window) - expected_start_date_2 = self.timedelta_formatted(self.start_date_2, days=0) # expected_conversion_window) + expected_primary_keys = self.expected_primary_keys()[stream] + expected_start_date_1 = self.start_date_1 + expected_start_date_2 = self.start_date_2 # collect information for assertions from syncs 1 & 2 base on expected values - # record_count_sync_1 = record_count_by_stream_1.get(stream, 0) # BUG_TDL-17535 - # record_count_sync_2 = record_count_by_stream_2.get(stream, 0) # BUG_TDL-17535 - primary_keys_list_1 = [tuple(message.get('data').get(expected_pk) for expected_pk in expected_primary_keys) - for message in synced_records_1.get(stream).get('messages') - if message.get('action') == 'upsert'] - primary_keys_list_2 = [tuple(message.get('data').get(expected_pk) for expected_pk in expected_primary_keys) - for message in synced_records_2.get(stream).get('messages') - if message.get('action') == 'upsert'] + record_count_sync_1 = record_count_by_stream_1.get(stream, 0) + record_count_sync_2 = record_count_by_stream_2.get(stream, 0) + primary_keys_list_1 = [tuple(message['data'][expected_pk] for expected_pk in expected_primary_keys) + for message in synced_records_1[stream]['messages'] + if message['action'] == 'upsert'] + primary_keys_list_2 = [tuple(message['data'][expected_pk] for expected_pk in expected_primary_keys) + for message in synced_records_2[stream]['messages'] + if message['action'] == 'upsert'] primary_keys_sync_1 = set(primary_keys_list_1) primary_keys_sync_2 = set(primary_keys_list_2) if self.is_report(stream): - # TODO IMPLEMENT WHEN REPORTS SYNC READY - # # collect information specific to incremental streams from syncs 1 & 2 - # expected_replication_key = next(iter(self.expected_replication_keys().get(stream))) - # replication_dates_1 =[row.get('data').get(expected_replication_key) for row in - # synced_records_1.get(stream, {'messages': []}).get('messages', []) - # if row.get('data')] - # replication_dates_2 =[row.get('data').get(expected_replication_key) for row in - # synced_records_2.get(stream, {'messages': []}).get('messages', []) - # if row.get('data')] - - # # # Verify replication key is greater or equal to start_date for sync 1 - # for replication_date in replication_dates_1: - # self.assertGreaterEqual( - # self.parse_date(replication_date), self.parse_date(expected_start_date_1), - # msg="Report pertains to a date prior to our start date.\n" + - # "Sync start_date: {}\n".format(expected_start_date_1) + - # "Record date: {} ".format(replication_date) - # ) - - # # Verify replication key is greater or equal to start_date for sync 2 - # for replication_date in replication_dates_2: - # self.assertGreaterEqual( - # self.parse_date(replication_date), self.parse_date(expected_start_date_2), - # msg="Report pertains to a date prior to our start date.\n" + - # "Sync start_date: {}\n".format(expected_start_date_2) + - # "Record date: {} ".format(replication_date) - # ) - - # # Verify the number of records replicated in sync 1 is greater than the number - # # of records replicated in sync 2 + # collect information specific to incremental streams from syncs 1 & 2 + expected_replication_key = next(iter(self.expected_replication_keys().get(stream))) + + replication_dates_1 = [row.get('data').get(expected_replication_key) for row in + synced_records_1.get(stream, {'messages': []}).get('messages', []) + if row.get('data')] + replication_dates_2 = [row.get('data').get(expected_replication_key) for row in + synced_records_2.get(stream, {'messages': []}).get('messages', []) + if row.get('data')] + + print(f"DATE BOUNDARIES SYNC 1: {stream} {sorted(replication_dates_1)[0]} {sorted(replication_dates_1)[-1]}") + # Verify replication key is greater or equal to start_date for sync 1 + expected_start_date = dt.strptime(expected_start_date_1, self.START_DATE_FORMAT) + for replication_date in replication_dates_1: + replication_date = dt.strptime(replication_date, self.REPLICATION_KEY_FORMAT) + self.assertGreaterEqual(replication_date, expected_start_date, + msg="Report pertains to a date prior to our start date.\n" + + "Sync start_date: {}\n".format(expected_start_date_1) + + "Record date: {} ".format(replication_date) + ) + + expected_start_date = dt.strptime(expected_start_date_2, self.START_DATE_FORMAT) + # Verify replication key is greater or equal to start_date for sync 2 + for replication_date in replication_dates_2: + replication_date = dt.strptime(replication_date, self.REPLICATION_KEY_FORMAT) + self.assertGreaterEqual(replication_date, expected_start_date, + msg="Report pertains to a date prior to our start date.\n" + + "Sync start_date: {}\n".format(expected_start_date_2) + + "Record date: {} ".format(replication_date) + ) + + # TODO Remove if this does not apply with the lookback window at the time that it is + # available as a configurable property. + # Verify the number of records replicated in sync 1 is greater than the number + # of records replicated in sync 2 # self.assertGreater(record_count_sync_1, record_count_sync_2) - # # Verify the records replicated in sync 2 were also replicated in sync 1 - # self.assertTrue(primary_keys_sync_2.issubset(primary_keys_sync_1)) - pass + # Verify the records replicated in sync 2 were also replicated in sync 1 + self.assertTrue(primary_keys_sync_2.issubset(primary_keys_sync_1)) + else: # Verify that the 2nd sync with a later start date (more recent) replicates # the same number of records as the 1st sync. - # self.assertEqual(record_count_sync_2, record_count_sync_1) # BUG_TDL-17535 + self.assertEqual(record_count_sync_2, record_count_sync_1) # Verify by primary key the same records are replicated in the 1st and 2nd syncs self.assertSetEqual(primary_keys_sync_1, primary_keys_sync_2) + +class StartDateTest1(StartDateTest): + + missing_coverage_streams = { # end result + 'display_keyword_performance_report', # no test data available + 'display_topics_performance_report', # no test data available + 'placement_performance_report', # no test data available + "keywords_performance_report", # no test data available + "keywordless_query_report", # no test data available + "video_performance_report", # no test data available + 'audience_performance_report', + "shopping_performance_report", + 'landing_page_report', + 'expanded_landing_page_report', + 'user_location_performance_report', + } + + def setUp(self): + self.start_date_1 = self.get_properties().get('start_date') # '2021-12-01T00:00:00Z', + self.start_date_2 = self.timedelta_formatted(self.start_date_1, days=15) + self.streams_to_test = self.expected_streams() - { + 'search_query_performance_report', # Covered in other start date test + } - self.missing_coverage_streams # TODO + + @staticmethod + def name(): + return "tt_google_ads_start_date" + + def test_run(self): + self.run_test() diff --git a/tests/test_google_ads_start_date_2.py b/tests/test_google_ads_start_date_2.py new file mode 100644 index 0000000..63e4750 --- /dev/null +++ b/tests/test_google_ads_start_date_2.py @@ -0,0 +1,22 @@ +import os +import unittest +from datetime import datetime as dt + +from tap_tester import connections, runner, menagerie + +from base import GoogleAdsBase +from test_google_ads_start_date import StartDateTest + + +class StartDateTest2(StartDateTest): + + def name(self): + return "tt_google_ads_start_date_2" + + def setUp(self): + self.start_date_1 = '2022-01-20T00:00:00Z' # '2022-01-25T00:00:00Z', + self.start_date_2 = self.timedelta_formatted(self.start_date_1, days=2) + self.streams_to_test = {'search_query_performance_report'} + + def test_run(self): + self.run_test() diff --git a/tests/test_google_ads_sync_canary.py b/tests/test_google_ads_sync_canary.py index 004dbd9..dbd02d5 100644 --- a/tests/test_google_ads_sync_canary.py +++ b/tests/test_google_ads_sync_canary.py @@ -7,7 +7,295 @@ class DiscoveryTest(GoogleAdsBase): - """Test tap discovery mode and metadata conforms to standards.""" + """ + Test tap's sync mode can extract records for all streams + with standard table and field selection. + """ + + @staticmethod + def expected_default_fields(): + """ + In this test core streams have all fields selected. + + Report streams will select fields based on the default values that + are provided when selecting the report type in Google's UI. + + returns a dictionary of reports to standard fields + """ + + # TODO_TDL-17909 [BUG?] commented out fields below are not discovered for the given stream by the tap + return { + 'ad_performance_report': { + # 'account_name', # 'Account name', + # 'ad_final_url', # 'Ad final URL', + # 'ad_group', # 'Ad group', + # 'ad_mobile_final_url', # 'Ad mobile final URL', + 'average_cpc', # 'Avg. CPC', + # 'business_name', # 'Business name', + # 'call_to_action_text', # 'Call to action text', + # 'campaign', # 'Campaign', + # 'campaign_subtype', # 'Campaign type', + # 'campaign_type', # 'Campaign subtype', + 'clicks', # 'Clicks', + 'conversions', # 'Conversions', + # 'conversion_rate', # 'Conv. rate', + # 'cost', # 'Cost', + 'cost_per_conversion', # 'Cost / conv.', + 'ctr', # 'CTR', + # 'currency_code', # 'Currency code', + 'customer_id', # 'Customer ID', + # 'description', # 'Description', + # 'description_1', # 'Description 1', + # 'description_2', # 'Description 2', + # 'description_3', # 'Description 3', + # 'description_4', # 'Description 4', + # 'final_url', # 'Final URL', + # 'headline_1', # 'Headline 1', + # 'headline_2', # 'Headline 2', + # 'headline_3', # 'Headline 3', + # 'headline_4', # 'Headline 4', + # 'headline_5', # 'Headline 5', + 'impressions', # 'Impr.', + # 'long_headline', # 'Long headline', + 'view_through_conversions', # 'View-through conv.', + }, + "adgroup_performance_report": { + # 'account_name', # Account name, + # 'ad_group', # Ad group, + # 'ad_group_state', # Ad group state, + 'average_cpc', # Avg. CPC, + # 'campaign', # Campaign, + # 'campaign_subtype', # Campaign subtype, + # 'campaign_type', # Campaign type, + 'clicks', # Clicks, + # 'conversion_rate', # Conv. rate + 'conversions', # Conversions, + # 'cost', # Cost, + 'cost_per_conversion', # Cost / conv., + 'ctr', # CTR, + # 'currency_code', # Currency code, + 'customer_id', # Customer ID, + 'impressions', # Impr., + 'view_through_conversions', # View-through conv., + }, + # TODO_TDL-17909 | [BUG?] missing audience fields + "audience_performance_report": { + # 'account_name', # Account name, + # 'ad_group_name', # 'ad_group', # Ad group, + # 'ad_group_default_max_cpc', # Ad group default max. CPC, + # 'audience_segment', # Audience segment, + # 'audience_segment_bid_adjustments', # Audience Segment Bid adj., + # 'audience_segment_max_cpc', # Audience segment max CPC, + # 'audience_segment_state', # Audience segment state, + 'average_cpc', # Avg. CPC, + 'average_cpm', # Avg. CPM + # 'campaign', # Campaign, + 'clicks', # Clicks, + # 'cost', # Cost, + 'ctr', # CTR, + # 'currency_code', # Currency code, + 'customer_id', # Customer ID, + 'impressions', # Impr., + 'ad_group_targeting_setting', # Targeting Setting, + }, + "campaign_performance_report": { + # 'account_name', # Account name, + 'average_cpc', # Avg. CPC, + # 'campaign', # Campaign, + # 'campaign_state', # Campaign state, + # 'campaign_type', # Campaign type, + 'clicks', # Clicks, + # 'conversion_rate', # Conv. rate + 'conversions', # Conversions, + # 'cost', # Cost, + 'cost_per_conversion', # Cost / conv., + 'ctr', # CTR, + # 'currency_code', # Currency code, + 'customer_id', # Customer ID, + 'impressions', # Impr., + 'view_through_conversions', # View-through conv., + }, + "click_performance_report": { + 'ad_group_ad', + 'ad_group_id', + 'ad_group_name', + 'ad_group_status', + 'ad_network_type', + 'area_of_interest', + 'campaign_location_target', + 'click_type', + 'clicks', + 'customer_descriptive_name', + 'customer_id', + 'device', + 'gclid', + 'location_of_presence', + 'month_of_year', + 'page_number', + 'slot', + 'user_list', + }, + "display_keyword_performance_report": { # TODO_TDL-17885 NO DATA AVAILABLE + # 'ad_group', # Ad group, + # 'ad_group_bid_strategy_type', # Ad group bid strategy type, + 'average_cpc', # Avg. CPC, + 'average_cpm', # Avg. CPM, + 'average_cpv', # Avg. CPV, + # 'campaign', # Campaign, + # 'campaign_bid_strategy_type', # Campaign bid strategy type, + # 'campaign_subtype', # Campaign subtype, + 'clicks', # Clicks, + # 'conversion_rate', # Conv. rate, + 'conversions', # Conversions, + # 'cost', # Cost, + 'cost_per_conversion', # Cost / conv., + # 'currency_code', # Currency code, + # 'display_video_keyword', # Display/video keyword, + 'impressions', # Impr., + 'interaction_rate', # Interaction rate, + 'interactions', # Interactions, + 'view_through_conversions', # View-through conv., + }, + "display_topics_performance_report": { # TODO_TDL-17885 NO DATA AVAILABLE + 'ad_group_name', # 'ad_group', # Ad group, + 'average_cpc', # Avg. CPC, + 'average_cpm', # Avg. CPM, + 'campaign_name', # 'campaign', # Campaign, + 'clicks', # Clicks, + # 'cost', # Cost, + 'ctr', # CTR, + 'customer_currency_code', # 'currency_code', # Currency code, + 'impressions', # Impr., + # 'topic', # Topic, + # 'topic_state', # Topic state, + }, + "placement_performance_report": { # TODO_TDL-17885 NO DATA AVAILABLE + # 'ad_group_name', + # 'ad_group_id', + # 'campaign_name', + # 'campaign_id', + 'clicks', + 'impressions', # Impr., + # 'cost', + 'ad_group_criterion_placement', # 'placement_group', 'placement_type', + }, + # "keywords_performance_report": set(), + # "shopping_performance_report": set(), + # BUG + "video_performance_report": { + # 'ad_group_name', + # 'all_conversions', + # 'all_conversions_from_interactions_rate', + # 'all_conversions_value', + # 'average_cpm', + # 'average_cpv', + 'campaign_name', + 'clicks', + # 'conversions', + # 'conversions_value', + # 'cost_per_all_conversions', + # 'cost_per_conversion', + # 'customer_descriptive_name', + # 'customer_id', + # 'impressions', + 'video_quartile_p25_rate', + # 'view_through_conversions', + }, + # NOTE AFTER THIS POINT COULDN"T FIND IN UI + "account_performance_report": { + 'average_cpc', + 'click_type', + 'clicks', + 'date', + 'descriptive_name', + 'id', + 'impressions', + 'invalid_clicks', + 'manager', + 'test_account', + 'time_zone', + }, + "geo_performance_report": { + 'clicks', + 'ctr', # CTR, + 'impressions', # Impr., + 'average_cpc', + # 'cost', + 'conversions', + 'view_through_conversions', # View-through conv., + 'cost_per_conversion', # Cost / conv., + # 'conversion_rate', # Conv. rate + # 'geo_target_city', + # 'geo_target_metro', + # 'geo_target_most_specific_location', + 'geo_target_region', + # 'country_criterion_id', # TODO_TDL-17910 | [BUG?] PROHIBITED_RESOURCE_TYPE_IN_SELECT_CLAUSE + }, + "gender_performance_report": { + # 'account_name', # Account name, + # 'ad_group', # Ad group, + # 'ad_group_state', # Ad group state, + 'ad_group_criterion_gender', + 'average_cpc', # Avg. CPC, + # 'campaign', # Campaign, + # 'campaign_subtype', # Campaign subtype, + # 'campaign_type', # Campaign type, + 'clicks', # Clicks, + # 'conversion_rate', # Conv. rate + 'conversions', # Conversions, + # 'cost', # Cost, + 'cost_per_conversion', # Cost / conv., + 'ctr', # CTR, + # 'currency_code', # Currency code, + 'customer_id', # Customer ID, + 'impressions', # Impr., + 'view_through_conversions', # View-through conv., + }, + "search_query_performance_report": { + 'clicks', + 'ctr', # CTR, + 'impressions', # Impr., + 'average_cpc', + # 'cost', + 'conversions', + 'view_through_conversions', # View-through conv., + 'cost_per_conversion', # Cost / conv., + # 'conversion_rate', # Conv. rate + 'search_term', + 'search_term_match_type', + }, + "age_range_performance_report": { + 'clicks', + 'ctr', # CTR, + 'impressions', # Impr., + 'average_cpc', + # 'cost', + 'conversions', + 'view_through_conversions', # View-through conv., + 'cost_per_conversion', # Cost / conv., + # 'conversion_rate', # Conv. rate + 'ad_group_criterion_age_range', # 'Age', + }, + 'placeholder_feed_item_report': { + 'clicks', + 'impressions', + 'placeholder_type', + }, + 'placeholder_report': { + # 'ad_group_id', + # 'ad_group_name', + # 'average_cpc', + # 'click_type', + 'clicks', + 'cost_micros', + # 'customer_descriptive_name', + # 'customer_id', + 'interactions', + 'placeholder_type', + }, + # 'landing_page_report': set(), # TODO_TDL-17885 + # 'expanded_landing_page_report': set(), # TODO_TDL-17885 + } @staticmethod def name(): @@ -17,51 +305,41 @@ def test_run(self): """ Testing that basic sync functions without Critical Errors """ - print("Discovery Test for tap-google-ads") + print("Canary Sync Test for tap-google-ads") conn_id = connections.ensure_connection(self) streams_to_test = self.expected_streams() - { - # TODO we are only testing core strems at the moment - 'landing_page_report', - 'expanded_landing_page_report', - 'display_topics_performance_report', - 'call_metrics_call_details_report', - 'gender_performance_report', - 'search_query_performance_report', - 'placeholder_feed_item_report', - 'keywords_performance_report', - 'video_performance_report', - 'campaign_performance_report', - 'geo_performance_report', - 'placeholder_report', - 'placement_performance_report', - 'click_performance_report', - 'display_keyword_performance_report', - 'shopping_performance_report', - 'ad_performance_report', - 'age_range_performance_report', - 'keywordless_query_report', - 'account_performance_report', - 'adgroup_performance_report', - 'audience_performance_report', + # TODO_TDL-17885 the following are not yet implemented + 'display_keyword_performance_report', # no test data available + 'display_topics_performance_report', # no test data available + 'audience_performance_report', # Potential BUG see above + 'placement_performance_report', # no test data available + "keywords_performance_report", # no test data available + "keywordless_query_report", # no test data available + "shopping_performance_report", # cannot find this in GoogleUI + "video_performance_report", # no test data available + "user_location_performance_report", # no test data available + 'landing_page_report', # not attempted + 'expanded_landing_page_report', # not attempted } # Run a discovery job - check_job_name = runner.run_check_mode(self, conn_id) - exit_status = menagerie.get_exit_status(conn_id, check_job_name) - menagerie.verify_check_exit_status(self, exit_status, check_job_name) + found_catalogs = self.run_and_verify_check_mode(conn_id) - # Verify a catalog was produced for each stream under test - found_catalogs = menagerie.get_catalogs(conn_id) - self.assertGreater(len(found_catalogs), 0) - found_catalog_names = {found_catalog['stream_name'] for found_catalog in found_catalogs} - self.assertSetEqual(streams_to_test, found_catalog_names) + # Perform table and field selection... + core_catalogs = [catalog for catalog in found_catalogs + if not self.is_report(catalog['stream_name']) + and catalog['stream_name'] in streams_to_test] + report_catalogs = [catalog for catalog in found_catalogs + if self.is_report(catalog['stream_name']) + and catalog['stream_name'] in streams_to_test] + # select all fields for core streams and... + self.select_all_streams_and_fields(conn_id, core_catalogs, select_all_fields=True) + # select 'default' fields for report streams + self.select_all_streams_and_default_fields(conn_id, report_catalogs) - # Perform table and field selection - self.select_all_streams_and_fields(conn_id, found_catalogs, select_all_fields=True) - - # Run a sync + # Run a sync sync_job_name = runner.run_sync_mode(self, conn_id) # Verify the tap and target do not throw a critical error @@ -73,6 +351,9 @@ def test_run(self): # Verify at least 1 record was replicated for each stream for stream in streams_to_test: - record_count = len(synced_records[stream]['messages']) - self.assertGreater(record_count, 0) + with self.subTest(stream=stream): + record_count = len(synced_records.get(stream, {'messages': []})['messages']) + self.assertGreater(record_count, 0) + print(f"{record_count} {stream} record(s) replicated.") + diff --git a/tests/unittests/test_resource_schema.py b/tests/unittests/test_resource_schema.py index 6d72e3d..7bf4f00 100644 --- a/tests/unittests/test_resource_schema.py +++ b/tests/unittests/test_resource_schema.py @@ -1,7 +1,7 @@ from collections import namedtuple import unittest -from tap_google_ads import get_segments -from tap_google_ads import get_attributes +from tap_google_ads.discover import get_segments +from tap_google_ads.discover import get_attributes RESOURCE_SCHEMA = { diff --git a/tests/unittests/test_utils.py b/tests/unittests/test_utils.py index 5f5fc0c..6916301 100644 --- a/tests/unittests/test_utils.py +++ b/tests/unittests/test_utils.py @@ -1,44 +1,216 @@ import unittest -from tap_google_ads.reports import flatten -from tap_google_ads.reports import make_field_names +from tap_google_ads.streams import generate_hash +from tap_google_ads.streams import get_query_date +from tap_google_ads.streams import create_nested_resource_schema +from singer import metadata +from singer.utils import strptime_to_utc +resource_schema = { + "accessible_bidding_strategy.id": {"json_schema": {"type": ["null", "integer"]}}, + "accessible_bidding_strategy.strategy.id": {"json_schema": {"type": ["null", "integer"]}} +} +class TestCreateNestedResourceSchema(unittest.TestCase): + def test_one(self): + actual = create_nested_resource_schema(resource_schema, ["accessible_bidding_strategy.id"]) + expected = { + "type": [ + "null", + "object" + ], + "properties": { + "accessible_bidding_strategy" : { + "type": ["null", "object"], + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + } + } + } + } + } + self.assertDictEqual(expected, actual) -class TestFlatten(unittest.TestCase): - def test_flatten_one_level(self): - nested_obj = {"a": {"b": "c"}, "d": "e"} - actual = flatten(nested_obj) - expected = {"a.b": "c", "d": "e"} + def test_two(self): + actual = create_nested_resource_schema(resource_schema, ["accessible_bidding_strategy.strategy.id"]) + expected = { + "type": ["null", "object"], + "properties": { + "accessible_bidding_strategy": { + "type": ["null", "object"], + "properties": { + "strategy": { + "type": ["null", "object"], + "properties": { + "id": {"type": ["null", "integer"]} + } + } + } + } + } + } self.assertDictEqual(expected, actual) - def test_flatten_two_levels(self): - nested_obj = {"a": {"b": {"c": "d", "e": "f"}, "g": "h"}} - actual = flatten(nested_obj) - expected = {"a.b.c": "d", "a.b.e": "f", "a.g": "h"} + def test_siblings(self): + actual = create_nested_resource_schema( + resource_schema, + ["accessible_bidding_strategy.id", "accessible_bidding_strategy.strategy.id"] + ) + expected = { + "type": ["null", "object"], + "properties": { + "accessible_bidding_strategy": { + "type": ["null", "object"], + "properties": { + "strategy": { + "type": ["null", "object"], + "properties": { + "id": {"type": ["null", "integer"]} + } + }, + "id": {"type": ["null", "integer"]} + } + } + } + } self.assertDictEqual(expected, actual) -class TestMakeFieldNames(unittest.TestCase): - def test_single_word(self): - actual = make_field_names("resource", ["type"]) - expected = ["resource.type"] - self.assertListEqual(expected, actual) +class TestRecordHashing(unittest.TestCase): + + test_record = { + 'id': 1234567890, + 'currency_code': 'USD', + 'time_zone': 'America/New_York', + 'auto_tagging_enabled': False, + 'manager': False, + 'test_account': False, + 'impressions': 0, + 'interactions': 0, + 'invalid_clicks': 0, + 'date': '2022-01-19', + } + + test_record_shuffled = { + 'currency_code': 'USD', + 'date': '2022-01-19', + 'auto_tagging_enabled': False, + 'time_zone': 'America/New_York', + 'test_account': False, + 'manager': False, + 'id': 1234567890, + 'interactions': 0, + 'invalid_clicks': 0, + 'impressions': 0, + } + + test_metadata = metadata.to_list({ + ('properties', 'id'): {'behavior': 'ATTRIBUTE'}, + ('properties', 'currency_code'): {'behavior': 'ATTRIBUTE'}, + ('properties', 'time_zone'): {'behavior': 'ATTRIBUTE'}, + ('properties', 'auto_tagging_enabled'): {'behavior': 'ATTRIBUTE'}, + ('properties', 'manager'): {'behavior': 'ATTRIBUTE'}, + ('properties', 'test_account'): {'behavior': 'ATTRIBUTE'}, + ('properties', 'impressions'): {'behavior': 'METRIC'}, + ('properties', 'interactions'): {'behavior': 'METRIC'}, + ('properties', 'invalid_clicks'): {'behavior': 'METRIC'}, + ('properties', 'date'): {'behavior': 'SEGMENT'}, + }) + + expected_hash = 'ade8240f134633fe125388e469e61ccf9e69033fd5e5f166b4b44766bc6376d3' + + def test_record_hash_canary(self): + self.assertEqual(self.expected_hash, generate_hash(self.test_record, self.test_metadata)) + + def test_record_hash_is_same_regardless_of_order(self): + self.assertEqual(self.expected_hash, generate_hash(self.test_record, self.test_metadata)) + self.assertEqual(self.expected_hash, generate_hash(self.test_record_shuffled, self.test_metadata)) + + def test_record_hash_is_same_with_fewer_metrics(self): + test_record_fewer_metrics = dict(self.test_record) + test_record_fewer_metrics.pop('interactions') + test_record_fewer_metrics.pop('invalid_clicks') + self.assertEqual(self.expected_hash, generate_hash(test_record_fewer_metrics, self.test_metadata)) + + def test_record_hash_is_different_with_non_metric_value(self): + test_diff_record = dict(self.test_record) + test_diff_record['date'] = '2022-02-03' + self.assertNotEqual(self.expected_hash, generate_hash(test_diff_record, self.test_metadata)) + + +class TestGetQueryDate(unittest.TestCase): + def test_one(self): + """Given: + - Start date before the conversion window + - No bookmark + + return the start date""" + actual = get_query_date( + start_date="2022-01-01T00:00:00Z", + bookmark=None, + conversion_window_date="2022-01-23T00:00:00Z" + ) + expected = strptime_to_utc("2022-01-01T00:00:00Z") + self.assertEqual(expected, actual) + + def test_two(self): + """Given: + - Start date before the conversion window + - bookmark after the conversion window + + return the conversion window""" + actual = get_query_date( + start_date="2022-01-01T00:00:00Z", + bookmark="2022-02-01T00:00:00Z", + conversion_window_date="2022-01-23T00:00:00Z" + ) + expected = strptime_to_utc("2022-01-23T00:00:00Z") + self.assertEqual(expected, actual) + + def test_three(self): + """Given: + - Start date after the conversion window + - no bookmark + + return the start date""" + actual = get_query_date( + start_date="2022-02-01T00:00:00Z", + bookmark=None, + conversion_window_date="2022-01-23T00:00:00Z" + ) + expected = strptime_to_utc("2022-02-01T00:00:00Z") + self.assertEqual(expected, actual) - def test_dotted_field(self): - actual = make_field_names("resource", ["tracking_setting.tracking_url"]) - expected = ["resource.tracking_setting.tracking_url"] - self.assertListEqual(expected, actual) + def test_four(self): + """Given: + - Start date after the conversion window + - bookmark after the start date - def test_foreign_key_field(self): - actual = make_field_names("resource", ["customer_id", "accessible_bidding_strategy_id"]) - expected = ["customer.id", "accessible_bidding_strategy.id"] - self.assertListEqual(expected, actual) + return the start date""" + actual = get_query_date( + start_date="2022-02-01T00:00:00Z", + bookmark="2022-02-08T00:00:00Z", + conversion_window_date="2022-01-23T00:00:00Z" + ) + expected = strptime_to_utc("2022-02-01T00:00:00Z") + self.assertEqual(expected, actual) - def test_trailing_id_field(self): - actual = make_field_names("resource", ["owner_customer_id"]) - expected = ["resource.owner_customer_id"] - self.assertListEqual(expected, actual) + def test_five(self): + """Given: + - Start date before the conversion window + - bookmark after the start date and before the conversion window + return the bookmark""" + actual = get_query_date( + start_date="2022-01-01T00:00:00Z", + bookmark="2022-01-14T00:00:00Z", + conversion_window_date="2022-01-23T00:00:00Z" + ) + expected = strptime_to_utc("2022-01-14T00:00:00Z") + self.assertEqual(expected, actual) if __name__ == '__main__': unittest.main()