From 1d9cdb6eb08dbdfb2af974d96539a74de0962eaf Mon Sep 17 00:00:00 2001 From: David Wallace Date: Tue, 7 May 2019 05:20:56 -0700 Subject: [PATCH] Remove Invalid Fields from Stream Discovery (#4) * remove max length constraint * bump to 0.0.5 * update base stream * update child object streams * bump to 0.0.6 * update metadata and stream properties * dump tap-framework dependency --- setup.py | 4 ++-- tap_chargebee/streams/addons.py | 3 ++- tap_chargebee/streams/base.py | 12 +++++------- tap_chargebee/streams/coupons.py | 3 ++- tap_chargebee/streams/credit_notes.py | 3 ++- tap_chargebee/streams/customers.py | 3 ++- tap_chargebee/streams/events.py | 3 ++- tap_chargebee/streams/invoices.py | 3 ++- tap_chargebee/streams/payment_sources.py | 3 ++- tap_chargebee/streams/plans.py | 3 ++- tap_chargebee/streams/subscriptions.py | 3 ++- tap_chargebee/streams/transactions.py | 3 ++- tap_chargebee/streams/virtual_bank_accounts.py | 3 ++- 13 files changed, 29 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index b96446c..7555631 100644 --- a/setup.py +++ b/setup.py @@ -3,13 +3,13 @@ from setuptools import setup, find_packages setup(name='tap-chargebee', - version='0.0.5', + version='0.0.6', description='Singer.io tap for extracting data from the Chargebee API', author='dwallace@envoy.com', classifiers=['Programming Language :: Python :: 3 :: Only'], py_modules=['tap_chargebee'], install_requires=[ - 'tap-framework==0.1.0' + 'tap-framework==0.1.1' ], entry_points=''' [console_scripts] diff --git a/tap_chargebee/streams/addons.py b/tap_chargebee/streams/addons.py index 8879200..97a030b 100644 --- a/tap_chargebee/streams/addons.py +++ b/tap_chargebee/streams/addons.py @@ -4,12 +4,13 @@ class AddonsStream(BaseChargebeeStream): TABLE = 'addons' ENTITY = 'addon' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/base.py b/tap_chargebee/streams/base.py index 8dbccb8..370b8b9 100644 --- a/tap_chargebee/streams/base.py +++ b/tap_chargebee/streams/base.py @@ -15,7 +15,7 @@ def write_schema(self): singer.write_schema( self.catalog.stream, self.catalog.schema.to_dict(), - key_properties=self.catalog.key_properties, + key_properties=self.KEY_PROPERTIES, bookmark_properties=self.BOOKMARK_PROPERTIES) def generate_catalog(self): @@ -23,11 +23,11 @@ def generate_catalog(self): mdata = singer.metadata.new() metadata = { - "selected": self.SELECTED, - "inclusion": self.INCLUSION, + "forced-replication-method": self.REPLICATION_METHOD, "valid-replication-keys": self.VALID_REPLICATION_KEYS, + "inclusion": self.INCLUSION, "selected-by-default": self.SELECTED_BY_DEFAULT, - "schema-name": self.TABLE + "table-key-properties": self.KEY_PROPERTIES } for k, v in metadata.items(): @@ -41,7 +41,7 @@ def generate_catalog(self): for field_name, field_schema in schema.get('properties').items(): inclusion = 'available' - if field_name in self.KEY_PROPERTIES: + if field_name in self.KEY_PROPERTIES or field_name in self.BOOKMARK_PROPERTIES: inclusion = 'automatic' mdata = singer.metadata.write( @@ -54,8 +54,6 @@ def generate_catalog(self): return [{ 'tap_stream_id': self.TABLE, 'stream': self.TABLE, - 'key_properties': self.KEY_PROPERTIES, - 'bookmark_properties': self.BOOKMARK_PROPERTIES, 'schema': self.get_schema(), 'metadata': singer.metadata.to_list(mdata) }] diff --git a/tap_chargebee/streams/coupons.py b/tap_chargebee/streams/coupons.py index 3c25b1e..30d3bd5 100644 --- a/tap_chargebee/streams/coupons.py +++ b/tap_chargebee/streams/coupons.py @@ -4,12 +4,13 @@ class CouponsStream(BaseChargebeeStream): TABLE = 'coupons' ENTITY = 'coupon' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/credit_notes.py b/tap_chargebee/streams/credit_notes.py index 4adbf35..092a871 100644 --- a/tap_chargebee/streams/credit_notes.py +++ b/tap_chargebee/streams/credit_notes.py @@ -4,12 +4,13 @@ class CreditNotesStream(BaseChargebeeStream): TABLE = 'credit_notes' ENTITY = 'credit_note' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/customers.py b/tap_chargebee/streams/customers.py index e0849da..6a51d9f 100644 --- a/tap_chargebee/streams/customers.py +++ b/tap_chargebee/streams/customers.py @@ -4,12 +4,13 @@ class CustomersStream(BaseChargebeeStream): TABLE = 'customers' ENTITY = 'customer' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/events.py b/tap_chargebee/streams/events.py index 80307b3..4c8a6d2 100644 --- a/tap_chargebee/streams/events.py +++ b/tap_chargebee/streams/events.py @@ -4,12 +4,13 @@ class EventsStream(BaseChargebeeStream): TABLE = 'events' ENTITY = 'event' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'occurred_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['occurred_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['occurred_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/invoices.py b/tap_chargebee/streams/invoices.py index 7ec576f..ed84367 100644 --- a/tap_chargebee/streams/invoices.py +++ b/tap_chargebee/streams/invoices.py @@ -4,12 +4,13 @@ class InvoicesStream(BaseChargebeeStream): TABLE = 'invoices' ENTITY = 'invoice' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/payment_sources.py b/tap_chargebee/streams/payment_sources.py index 9929504..8b1d6b0 100644 --- a/tap_chargebee/streams/payment_sources.py +++ b/tap_chargebee/streams/payment_sources.py @@ -4,12 +4,13 @@ class PaymentSourcesStream(BaseChargebeeStream): TABLE = 'payment_sources' ENTITY = 'payment_source' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/plans.py b/tap_chargebee/streams/plans.py index a413e20..de62953 100644 --- a/tap_chargebee/streams/plans.py +++ b/tap_chargebee/streams/plans.py @@ -4,12 +4,13 @@ class PlansStream(BaseChargebeeStream): TABLE = 'plans' ENTITY = 'plan' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/subscriptions.py b/tap_chargebee/streams/subscriptions.py index b6d6499..5f04033 100644 --- a/tap_chargebee/streams/subscriptions.py +++ b/tap_chargebee/streams/subscriptions.py @@ -4,12 +4,13 @@ class SubscriptionsStream(BaseChargebeeStream): TABLE = 'subscriptions' ENTITY = 'subscription' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/transactions.py b/tap_chargebee/streams/transactions.py index 7ee0636..a287604 100644 --- a/tap_chargebee/streams/transactions.py +++ b/tap_chargebee/streams/transactions.py @@ -4,12 +4,13 @@ class TransactionsStream(BaseChargebeeStream): TABLE = 'transactions' ENTITY = 'transaction' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self): diff --git a/tap_chargebee/streams/virtual_bank_accounts.py b/tap_chargebee/streams/virtual_bank_accounts.py index e8a78ea..edabb90 100644 --- a/tap_chargebee/streams/virtual_bank_accounts.py +++ b/tap_chargebee/streams/virtual_bank_accounts.py @@ -4,12 +4,13 @@ class VirtualBankAccountsStream(BaseChargebeeStream): TABLE = 'virtual_bank_accounts' ENTITY = 'virtual_bank_account' + REPLICATION_METHOD = 'INCREMENTAL' + REPLICATION_KEY = 'updated_at' KEY_PROPERTIES = ['id'] BOOKMARK_PROPERTIES = ['updated_at'] SELECTED_BY_DEFAULT = True VALID_REPLICATION_KEYS = ['updated_at'] INCLUSION = 'available' - SELECTED = True API_METHOD = 'GET' def get_url(self):