Hack: Handle specs that contain boolean fields that are required. (#929)

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2470e835-feaf-4db6-96f3-70fd645acc77.json

@@ -2,6 +2,6 @@
   "sourceDefinitionId": "2470e835-feaf-4db6-96f3-70fd645acc77",
   "name": "Salesforce",
   "dockerRepository": "airbyte/source-salesforce-singer",
-  "dockerImageTag": "0.1.0",
+  "dockerImageTag": "0.1.1",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-salesforce-singer"
 }

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/74d47f79-8d01-44ac-9755-f5eb0d7caacb.json

@@ -2,6 +2,6 @@
   "sourceDefinitionId": "74d47f79-8d01-44ac-9755-f5eb0d7caacb",
   "name": "Facebook Marketing APIs",
   "dockerRepository": "airbyte/source-facebook-marketing-api-singer",
-  "dockerImageTag": "0.1.0",
+  "dockerImageTag": "0.1.1",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-facebook-marketing-api-singer"
 }

airbyte-integrations/connectors/source-facebook-marketing-api-singer/Dockerfile

@@ -13,5 +13,5 @@ COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
 RUN pip install ".[main]"
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-facebook-marketing-api-singer

airbyte-integrations/connectors/source-facebook-marketing-api-singer/source_facebook_marketing_api_singer/source.py

@@ -34,12 +34,13 @@ def __init__(self):
         super().__init__()
 
     def transform_config(self, raw_config):
+        # todo (cgardens) - this is supposed to be handled in the ui and the api but neither of them are able to handle it right now. issue: https://github.com/airbytehq/airbyte/issues/892
         return {
             "start_date": raw_config["start_date"],
             "account_id": raw_config["account_id"],
             "access_token": raw_config["access_token"],
             # tap-singer expects a string not a boolean
-            "include_deleted": str(raw_config.get("include_deleted", False)),
+            "include_deleted": str(raw_config.get("include_deleted", True)),
         }
 
     def check(self, logger: AirbyteLogger, config_container: ConfigContainer) -> AirbyteConnectionStatus:

airbyte-integrations/connectors/source-facebook-marketing-api-singer/unit_tests/unit_test.py

@@ -39,3 +39,14 @@ def test_discover_cmd():
 def test_read_cmd():
     cmd = CONNECTOR.read_cmd(LOGGER, FAKE_CONFIG_PATH, FAKE_CATALOG_PATH)
     assert f"tap-facebook -c {FAKE_CONFIG_PATH} -p {FAKE_CATALOG_PATH}" == cmd.strip()
+
+
+def test_transform_config_adds_include_deleted_sandbox_if_empty():
+    input = {
+        "start_date": "start_date",
+        "account_id": "account_id",
+        "access_token": "access_token",
+    }
+    expected = dict(input)
+    expected["include_deleted"] = "True"
+    assert expected == CONNECTOR.transform_config(input)

airbyte-integrations/connectors/source-salesforce-singer/Dockerfile

@@ -8,7 +8,7 @@ ENV CODE_PATH="source_salesforce_singer"
 ENV AIRBYTE_IMPL_MODULE="source_salesforce_singer"
 ENV AIRBYTE_IMPL_PATH="SourceSalesforceSinger"
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-salesforce-singer
 
 WORKDIR /airbyte/integration_code

airbyte-integrations/connectors/source-salesforce-singer/requirements.txt

@@ -1,3 +1,4 @@
 -e ../../bases/airbyte-protocol
+-e ../../bases/base-python
 -e ../../bases/base-singer
 -e .

airbyte-integrations/connectors/source-salesforce-singer/setup.py

@@ -35,6 +35,7 @@
         "tap-salesforce==1.4.34",
         "requests",
         "airbyte-protocol",
+        "base-python",
         "base-singer",
     ],
 )

airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/source.py

@@ -85,5 +85,10 @@ def read_cmd(self, logger, config_path, catalog_path, state_path=None) -> str:
     def transform_config(self, raw_config):
         # the select_fields_by_default is opinionated about schema changes. we want to reserve the right for the Airbyte system to handle these changes, instead of the singer source.
         rendered_config = dict(raw_config)
+
+        # todo (cgardens) - this is supposed to be handled in the ui and the api but neither of them are able to handle it right now. issue: https://github.com/airbytehq/airbyte/issues/892
+        if "is_sandbox" not in raw_config:
+            rendered_config["is_sandbox"] = False
+
         rendered_config["select_fields_by_default"] = True
         return rendered_config

airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/spec.json

@@ -10,7 +10,6 @@
       "client_secret",
       "refresh_token",
       "start_date",
-      "is_sandbox",
       "api_type"
     ],
     "additionalProperties": false,

airbyte-integrations/connectors/source-salesforce-singer/unit_tests/unit_test.py

@@ -0,0 +1,31 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+from source_salesforce_singer import SourceSalesforceSinger
+
+CONNECTOR = SourceSalesforceSinger()
+
+
+def test_transform_config_adds_is_sandbox_if_empty():
+    assert {"is_sandbox": False, "select_fields_by_default": True} == CONNECTOR.transform_config({})