Remove Invalid Fields from Stream Discovery (#4)

* remove max length constraint

* bump to 0.0.5

* update base stream

* update child object streams

* bump to 0.0.6

* update metadata and stream properties

* dump tap-framework dependency

setup.py

@@ -3,13 +3,13 @@
 from setuptools import setup, find_packages
 
 setup(name='tap-chargebee',
-      version='0.0.5',
+      version='0.0.6',
       description='Singer.io tap for extracting data from the Chargebee API',
       author='[email protected]',
       classifiers=['Programming Language :: Python :: 3 :: Only'],
       py_modules=['tap_chargebee'],
       install_requires=[
-          'tap-framework==0.1.0'
+          'tap-framework==0.1.1'
       ],
       entry_points='''
           [console_scripts]

tap_chargebee/streams/addons.py

@@ -4,12 +4,13 @@
 class AddonsStream(BaseChargebeeStream):
     TABLE = 'addons'
     ENTITY = 'addon'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/base.py

@@ -15,19 +15,19 @@ def write_schema(self):
         singer.write_schema(
             self.catalog.stream,
             self.catalog.schema.to_dict(),
-            key_properties=self.catalog.key_properties,
+            key_properties=self.KEY_PROPERTIES,
             bookmark_properties=self.BOOKMARK_PROPERTIES)
 
     def generate_catalog(self):
         schema = self.get_schema()
         mdata = singer.metadata.new()
 
         metadata = {
-            "selected": self.SELECTED,
-            "inclusion": self.INCLUSION,
+            "forced-replication-method": self.REPLICATION_METHOD,
             "valid-replication-keys": self.VALID_REPLICATION_KEYS,
+            "inclusion": self.INCLUSION,
             "selected-by-default": self.SELECTED_BY_DEFAULT,
-            "schema-name": self.TABLE
+            "table-key-properties": self.KEY_PROPERTIES
         }
 
         for k, v in metadata.items():
@@ -41,7 +41,7 @@ def generate_catalog(self):
         for field_name, field_schema in schema.get('properties').items():
             inclusion = 'available'
 
-            if field_name in self.KEY_PROPERTIES:
+            if field_name in self.KEY_PROPERTIES or field_name in self.BOOKMARK_PROPERTIES:
                 inclusion = 'automatic'
 
             mdata = singer.metadata.write(
@@ -54,8 +54,6 @@ def generate_catalog(self):
         return [{
             'tap_stream_id': self.TABLE,
             'stream': self.TABLE,
-            'key_properties': self.KEY_PROPERTIES,
-            'bookmark_properties': self.BOOKMARK_PROPERTIES,
             'schema': self.get_schema(),
             'metadata': singer.metadata.to_list(mdata)
         }]

tap_chargebee/streams/coupons.py

@@ -4,12 +4,13 @@
 class CouponsStream(BaseChargebeeStream):
     TABLE = 'coupons'
     ENTITY = 'coupon'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/credit_notes.py

@@ -4,12 +4,13 @@
 class CreditNotesStream(BaseChargebeeStream):
     TABLE = 'credit_notes'
     ENTITY = 'credit_note'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/customers.py

@@ -4,12 +4,13 @@
 class CustomersStream(BaseChargebeeStream):
     TABLE = 'customers'
     ENTITY = 'customer'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/events.py

@@ -4,12 +4,13 @@
 class EventsStream(BaseChargebeeStream):
     TABLE = 'events'
     ENTITY = 'event'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'occurred_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['occurred_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['occurred_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/invoices.py

@@ -4,12 +4,13 @@
 class InvoicesStream(BaseChargebeeStream):
     TABLE = 'invoices'
     ENTITY = 'invoice'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/payment_sources.py

@@ -4,12 +4,13 @@
 class PaymentSourcesStream(BaseChargebeeStream):
     TABLE = 'payment_sources'
     ENTITY = 'payment_source'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/plans.py

@@ -4,12 +4,13 @@
 class PlansStream(BaseChargebeeStream):
     TABLE = 'plans'
     ENTITY = 'plan'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/subscriptions.py

@@ -4,12 +4,13 @@
 class SubscriptionsStream(BaseChargebeeStream):
     TABLE = 'subscriptions'
     ENTITY = 'subscription'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/transactions.py

@@ -4,12 +4,13 @@
 class TransactionsStream(BaseChargebeeStream):
     TABLE = 'transactions'
     ENTITY = 'transaction'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):

tap_chargebee/streams/virtual_bank_accounts.py

@@ -4,12 +4,13 @@
 class VirtualBankAccountsStream(BaseChargebeeStream):
     TABLE = 'virtual_bank_accounts'
     ENTITY = 'virtual_bank_account'
+    REPLICATION_METHOD = 'INCREMENTAL'
+    REPLICATION_KEY = 'updated_at'
     KEY_PROPERTIES = ['id']
     BOOKMARK_PROPERTIES = ['updated_at']
     SELECTED_BY_DEFAULT = True
     VALID_REPLICATION_KEYS = ['updated_at']
     INCLUSION = 'available'
-    SELECTED = True
     API_METHOD = 'GET'
 
     def get_url(self):