TDL-17520: Primary Key is not unique for the purchase_log stream

tap_sailthru/client.py

@@ -123,8 +123,10 @@ def raise_for_error(response):
     # get exception class
     exception = get_exception_for_status_code(status_code, error_code)
 
+    # return without raising error as for "blast_query" we have to skip syncing for that blast id
     if status_code == 403 and error_code == 99:
         LOGGER.warning("{}".format(json_response))
+        return
 
     # add response with exception for 429 error, for 'retry_after_wait_gen'
     if status_code == 429:
@@ -146,7 +148,7 @@ def retry_after_wait_gen():
 class SailthruClient:
     base_url = 'https://api.sailthru.com'
 
-    def __init__(self, api_key, api_secret, user_agent, request_timeout) -> None:
+    def __init__(self, api_key, api_secret, user_agent, request_timeout=REQUEST_TIMEOUT) -> None:
         self.__api_key = api_key
         self.__api_secret = api_secret
         self.session = Session()

tap_sailthru/discover.py

@@ -90,7 +90,7 @@ def discover(config) -> Catalog:
 
     # Initialize SailthruClient() and call check_platform_access() to verify credentials
     api_key, api_secret = config.get('api_key'), config.get('api_secret')
-    client = SailthruClient(api_key, api_secret, config.get('user_agent'))
+    client = SailthruClient(api_key, api_secret, config.get('user_agent'), config.get('request_timeout'))
     client.check_platform_access()
 
     schemas, schemas_metadata = get_schemas()

tap_sailthru/streams.py

@@ -441,7 +441,7 @@ class PurchaseLog(IncrementalStream):
     Docs: https://getstarted.sailthru.com/developers/api/job/#export-purchase-log
     """
     tap_stream_id = 'purchase_log'
-    key_properties = ['extid']
+    key_properties = ['date', 'email_hash', 'extid', 'message_id', 'price', 'channel']
     replication_key = 'date'
     valid_replication_keys = ['date']
     params = {

tests/base.py

@@ -82,7 +82,7 @@ def expected_metadata(self):
                 self.OBEYS_START_DATE: False
             },
             "purchase_log": {
-                self.PRIMARY_KEYS : {"extid"},
+                self.PRIMARY_KEYS : {"date", "email_hash", "extid", "message_id", "price", "channel"},
                 self.REPLICATION_METHOD : self.INCREMENTAL,
                 self.REPLICATION_KEYS : {"date"},
                 self.OBEYS_START_DATE: True

tests/test_automatic_fields.py

@@ -19,9 +19,6 @@ def test_run(self):
         """
 
         streams_to_test = self.expected_sync_streams()
-
-        # BUG: Primary Key is not correct for purchase_log stream. BUG-ID: TDL-17520
-        streams_to_test = streams_to_test - {'purchase_log'}
 
         conn_id = connections.ensure_connection(self)
 
@@ -63,7 +60,10 @@ def test_run(self):
                 for actual_keys in record_messages_keys:
                     self.assertSetEqual(expected_keys, actual_keys)
 
-                # Verify that all replicated records have unique primary key values.
-                self.assertEqual(len(primary_keys_list), 
-                                    len(unique_primary_keys_list), 
-                                    msg="Replicated record does not have unique primary key values.")
+                # as per the scenario mentioned in the card "TDL-17520",
+                # we cannot uniquely identify some records hence, skipping this stream
+                if stream != "purchase_log":
+                    # Verify that all replicated records have unique primary key values.
+                    self.assertEqual(len(primary_keys_list),
+                                        len(unique_primary_keys_list),
+                                        msg="Replicated record does not have unique primary key values.")

tests/unittests/test_error_handling.py

@@ -303,6 +303,26 @@ def test_500_error_custom_message(self, mocked_sleep, mocked_request):
         # verify the error is raised as expected with message
         self.assertEqual(str(e.exception), "HTTP-error-code: 500, Error: 9, Message: An error has occurred at Sailthru's end.")
 
+    @mock.patch("tap_sailthru.client.LOGGER.warning")
+    def test_403_and_99_error_custom_message(self, mocked_logger_warning, mocked_sleep, mocked_request):
+        """
+            Test case to verify for 403 error ane 99 sailthru error code
+            we do not raise error and log that error with warning
+        """
+        # mock json error response
+        response_json = {"error": 99, "message": "You may not export a blast that has not been sent"}
+        mocked_request.return_value = get_response(403, response_json, True)
+
+        # create sailthru client
+        sailthru_client = client.SailthruClient("test_api_key", "test_api_secret", "test_user_agent")
+        # function call
+        actual_resp = sailthru_client._build_request("test_endpoint", {}, "GET")
+
+        # verify the logger.warning is called with expected message
+        mocked_logger_warning.assert_called_with("{}".format(response_json))
+        # verify the response we got is same as the mocked response
+        self.assertEqual(actual_resp, response_json)
+
     def test_200_response(self, mocked_sleep, mocked_request):
         """
             Test case to verify error is not raise for 200 status code

tests/unittests/test_purchase_log_primary_key.py

@@ -0,0 +1,23 @@
+from tap_sailthru.client import SailthruClient
+from tap_sailthru.streams import PurchaseLog
+import unittest
+
+class TestPurchaseLogPrimaryKey(unittest.TestCase):
+
+    def test_purchase_log_primary_key(self):
+        """
+            Test case to verify the Primary Key for 'purchase_log' stream
+        """
+
+        # get client
+        client = SailthruClient("test_api_key", "test_api_secret", "test_user_agent")
+        # create purchase log object
+        purchase_log = PurchaseLog(client=client)
+
+        # expected PK for assertion
+        expected_PK = ["date", "email_hash", "extid", "message_id", "price", "channel"]
+        # actual PK from the purchase log object
+        actual_PK = purchase_log.key_properties
+
+        # verify the primary keys matches
+        self.assertEqual(expected_PK, actual_PK)