feat(gsheets2): added hidden properties

doc/connectors/google_sheets_2.md

@@ -4,16 +4,16 @@
 
 * `type`: `"GoogleSheets2"`
 * `name`: str, required
-* `auth_flow`: str
+* `_auth_flow`: str
 * `auth_flow_id`: str
-* `baseroute`: str
-* `secrets`: dict
+* `_baseroute`: str
+* `hidden_properties`: GoogleSheets2HiddenProperties
 
-The `auth_flow` property marks this as being a connector that uses the connector_oauth_manager for the oauth dance.
+The `_auth_flow` property marks this as being a connector that uses the connector_oauth_manager for the oauth dance. It is hidden from being rendered in the front but appears in the front as a flag for initiating the oauth dance. It is a static *flag*.
 
-The `baseroute` is fixed and is 'https://sheets.googleapis.com/v4/spreadsheets/'.
+The `_baseroute` is fixed and is 'https://sheets.googleapis.com/v4/spreadsheets/'. It too is hidden from being rendered in the front. It is a static *non-flag*
 
-The `secrets` dictionary contains the `access_token` and a `refresh_token` (if there is one). Though `secrets` is optional during the initial creation of the connector, it is necessary for when the user wants to make requests to the connector. If there is no `access_token`, an Exception is thrown.
+The `hidden_properties` propertie contains the `access_token` and a `refresh_token` (if there is one) in a class called `GoogleSheets2HiddenProperties`. Though `hidden_properties` is optional during the initial creation of the connector, it is necessary for when the user wants to make requests to the connector. If there is no `access_token`, an Exception is thrown. This is a dynamic *non-flag* property.
 
 The `auth_flow_id` property is like an identifier that is used to identify the secrets associated with the connector.
 
@@ -22,6 +22,7 @@ The `auth_flow_id` property is like an identifier that is used to identify the s
 DATA_PROVIDERS: [
   type:    'GoogleSheets'
   name:    '<name>'
+  auth_flow_id: '<auth_flow_id>'
 ,
   ...
 ]

tests/google_sheets_2/test_google_sheets_2.py

@@ -8,19 +8,22 @@
 from toucan_connectors.google_sheets_2.google_sheets_2_connector import (
     GoogleSheets2Connector,
     GoogleSheets2DataSource,
+    GoogleSheets2HiddenProperties,
 )
 
 import_path = 'toucan_connectors.google_sheets_2.google_sheets_2_connector'
 
 
 @fixture
 def con():
-    return GoogleSheets2Connector(name='test_name')
+    con = GoogleSheets2Connector(name='test_name', auth_flow_id='bar')
+    con.set_hidden_properties(secrets=None)
+    return con
 
 
 @fixture
-def con_with_secrets(con):
-    con.set_secrets({'access_token': 'foo', 'refresh_token': None})
+def con_with_hidden_properties(con):
+    con.set_hidden_properties(secrets={'access_token': 'foo', 'refresh_token': None})
     return con
 
 
@@ -85,12 +88,12 @@ def get_columns_in_schema(schema):
         return None
 
 
-def test_get_form_with_secrets(mocker, con_with_secrets, ds):
+def test_get_form_with_secrets(mocker, con_with_hidden_properties, ds):
     """It should return a list of spreadsheet titles."""
     mocker.patch.object(GoogleSheets2Connector, '_run_fetch', return_value=FAKE_SHEET_LIST_RESPONSE)
 
     result = ds.get_form(
-        connector=con_with_secrets,
+        connector=con_with_hidden_properties,
         current_config={'spreadsheet_id': '1SMnhnmBm-Tup3SfhS03McCf6S4pS2xqjI6CAXSSBpHU'},
     )
     expected_results = ['Foo', 'Bar', 'Baz']
@@ -107,30 +110,37 @@ def test_get_form_no_secrets(mocker, con, ds):
     assert not get_columns_in_schema(result)
 
 
-def test_set_secrets(mocker, con):
-    """It should set secrets on the connector."""
-    spy = mocker.spy(GoogleSheets2Connector, 'set_secrets')
+def test_set_hidden_properties(mocker, con):
+    """It should set secrets in the connector's hidden properties."""
+    spy = mocker.spy(GoogleSheets2HiddenProperties, 'set_hidden_properties')
     fake_secrets = {
         'access_token': 'myaccesstoken',
         'refresh_token': None,
     }
-    con.set_secrets(fake_secrets)
+    con.set_hidden_properties(secrets=fake_secrets)
 
-    assert con.secrets == fake_secrets
-    spy.assert_called_once_with(con, fake_secrets)
+    assert con.hidden_properties.get_hidden_properties() == {
+        'secrets': fake_secrets,
+    }
+    spy.assert_called_once_with(
+        GoogleSheets2HiddenProperties(
+            secrets={'access_token': 'myaccesstoken', 'refresh_token': None}
+        ),
+        hidden_properties={'secrets': fake_secrets},
+    )
 
 
-def test_spreadsheet_success(mocker, con_with_secrets, ds):
+def test_spreadsheet_success(mocker, con_with_hidden_properties, ds):
     """It should return a spreadsheet."""
     mocker.patch.object(GoogleSheets2Connector, '_run_fetch', return_value=FAKE_SHEET)
 
-    df = con_with_secrets.get_df(ds)
+    df = con_with_hidden_properties.get_df(ds)
 
     assert df.shape == (2, 2)
     assert df.columns.tolist() == ['country', 'city']
 
     ds.header_row = 1
-    df = con_with_secrets.get_df(ds)
+    df = con_with_hidden_properties.get_df(ds)
     assert df.shape == (1, 2)
     assert df.columns.tolist() == ['France', 'Paris']
 
@@ -144,21 +154,21 @@ def test_spreadsheet_no_secrets(mocker, con, ds):
 
     assert str(err.value) == 'No credentials'
 
-    con.set_secrets({'refresh_token': None})
+    con.set_hidden_properties(secrets={'refresh_token': None})
 
     with pytest.raises(KeyError):
         con.get_df(ds)
 
 
-def test_set_columns(mocker, con_with_secrets, ds):
+def test_set_columns(mocker, con_with_hidden_properties, ds):
     """It should return a well-formed column set."""
     fake_results = {
         'metadata': '...',
         'values': [['Animateur', '', '', 'Week'], ['pika', '', 'a', 'W1'], ['bulbi', '', '', 'W2']],
     }
     mocker.patch.object(GoogleSheets2Connector, '_run_fetch', return_value=fake_results)
 
-    df = con_with_secrets.get_df(ds)
+    df = con_with_hidden_properties.get_df(ds)
     assert df.to_dict() == {
         'Animateur': {1: 'pika', 2: 'bulbi'},
         1: {1: '', 2: ''},
@@ -178,7 +188,7 @@ def test__run_fetch(mocker, con):
     assert result == FAKE_SHEET
 
 
-def test_spreadsheet_without_sheet(mocker, con_with_secrets, ds_without_sheet):
+def test_spreadsheet_without_sheet(mocker, con_with_hidden_properties, ds_without_sheet):
     """
     It should retrieve the first sheet of the spreadsheet if no sheet has been indicated
     """
@@ -192,7 +202,7 @@ def mock_api_responses(uri: str, _token):
     fetch_mock: Mock = mocker.patch.object(
         GoogleSheets2Connector, '_run_fetch', side_effect=mock_api_responses
     )
-    df = con_with_secrets.get_df(ds_without_sheet)
+    df = con_with_hidden_properties.get_df(ds_without_sheet)
 
     assert fetch_mock.call_count == 2
     assert (
@@ -215,15 +225,15 @@ def test_get_status_no_secrets(mocker, con):
     assert con.get_status().status is False
 
 
-def test_get_status_success(mocker, con_with_secrets):
+def test_get_status_success(mocker, con_with_hidden_properties):
     """
     It should fail if no secrets are provided
     """
     fetch_mock: Mock = mocker.patch.object(
         GoogleSheets2Connector, '_run_fetch', return_value={'email': '[email protected]'}
     )
 
-    connector_status = con_with_secrets.get_status()
+    connector_status = con_with_hidden_properties.get_status()
     assert connector_status.status is True
     assert '[email protected]' in connector_status.message
 
@@ -232,10 +242,10 @@ def test_get_status_success(mocker, con_with_secrets):
     )
 
 
-def test_get_status_api_down(mocker, con_with_secrets):
+def test_get_status_api_down(mocker, con_with_hidden_properties):
     """
     It should fail if no secrets are provided
     """
     mocker.patch.object(GoogleSheets2Connector, '_run_fetch', side_effect=HttpError)
 
-    assert con_with_secrets.get_status().status is False
+    assert con_with_hidden_properties.get_status().status is False

toucan_connectors/__init__.py

@@ -194,7 +194,7 @@ def html_base64_image_src(image_path: str) -> str:
         with suppress(AttributeError):
             connector_infos['bearer_integration'] = connector_cls.bearer_integration
         with suppress(AttributeError):
-            connector_infos['auth_flow'] = connector_cls.auth_flow
+            connector_infos['_auth_flow'] = connector_cls._auth_flow
         # check if connector implements `get_status`,
         # which is hence different from `ToucanConnector.get_status`
         connector_infos['hasStatusCheck'] = (

toucan_connectors/google_sheets_2/google_sheets_2_connector.py

@@ -7,7 +7,7 @@
 
 import pandas as pd
 from aiohttp import ClientSession
-from pydantic import Field, create_model
+from pydantic import BaseModel, Field, create_model
 
 from toucan_connectors.common import ConnectorStatus, HttpError, fetch, get_loop
 from toucan_connectors.toucan_connector import ToucanConnector, ToucanDataSource, strlist_to_enum
@@ -43,8 +43,11 @@ def get_form(cls, connector: 'GoogleSheets2Connector', current_config):
         constraints = {}
         with suppress(Exception):
             partial_endpoint = current_config['spreadsheet_id']
-            final_url = f'{connector.baseroute}{partial_endpoint}'
-            data = connector._run_fetch(final_url, connector.secrets['access_token'])
+            final_url = f'{connector._baseroute}{partial_endpoint}'
+            data = connector._run_fetch(
+                final_url,
+                connector.hidden_properties.get_hidden_properties()['secrets']['access_token'],
+            )
             available_sheets = [str(x['properties']['title']) for x in data['sheets']]
             constraints['sheet'] = strlist_to_enum('sheet', available_sheets)
 
@@ -54,20 +57,41 @@ def get_form(cls, connector: 'GoogleSheets2Connector', current_config):
 Secrets = Dict[str, Any]
 
 
+class GoogleSheets2HiddenProperties(BaseModel):
+    """
+    This class contains dynamic hidden properties for Google Sheets 2 that are not flags
+
+    - secrets
+    """
+
+    secrets: Optional[Secrets]
+
+    def get_hidden_properties(self):
+        """Get hidden properties stored in this class."""
+        return {k: v for k, v in self.__dict__.items()}
+
+    def set_hidden_properties(self, hidden_properties):
+        """Set hidden properties to be stored in this class."""
+        for k, v in hidden_properties.items():
+            if hasattr(self, k):
+                setattr(self, k, v)
+
+
 class GoogleSheets2Connector(ToucanConnector):
     """The Google Sheets connector."""
 
+    # dynamic, non-hidden variables
     data_source_model: GoogleSheets2DataSource
-
-    auth_flow = 'oauth2'
-
     auth_flow_id: Optional[str]
 
-    # The following should be hidden properties
+    # flagging variable; required for initiating the oauth dance
+    _auth_flow = 'oauth2'
 
-    baseroute = 'https://sheets.googleapis.com/v4/spreadsheets/'
+    # static variable
+    _baseroute = 'https://sheets.googleapis.com/v4/spreadsheets/'
 
-    secrets: Optional[Secrets]
+    # hidden dynamic variables
+    hidden_properties: Optional[GoogleSheets2HiddenProperties]
 
     async def _authentified_fetch(self, url, access_token):
         """Build the final request along with headers."""
@@ -76,9 +100,11 @@ async def _authentified_fetch(self, url, access_token):
         async with ClientSession(headers=headers) as session:
             return await fetch(url, session)
 
-    def set_secrets(self, secrets: Secrets):
-        """Set the secrets from inside the main service."""
-        self.secrets = secrets
+    def set_hidden_properties(self, secrets: Secrets):
+        """Set the hidden properties from inside the main service."""
+        hidden_properties = GoogleSheets2HiddenProperties()
+        hidden_properties.set_hidden_properties(hidden_properties={'secrets': secrets})
+        self.hidden_properties = hidden_properties
 
     def _run_fetch(self, url, access_token):
         """Run loop."""
@@ -93,21 +119,23 @@ def _retrieve_data(self, data_source: GoogleSheets2DataSource) -> pd.DataFrame:
         Requires:
         - Datasource
         """
-        if not self.secrets:
+        hidden_properties = self.hidden_properties.get_hidden_properties()
+        secrets = hidden_properties.get('secrets')
+        if hidden_properties is None or secrets is None:
             raise Exception('No credentials')
 
-        access_token = self.secrets['access_token']
+        access_token = secrets['access_token']
 
         if data_source.sheet is None:
             # Get spreadsheet informations and retrieve all the available sheets
             # https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/get
-            data = self._run_fetch(f'{self.baseroute}{data_source.spreadsheet_id}', access_token)
+            data = self._run_fetch(f'{self._baseroute}{data_source.spreadsheet_id}', access_token)
             available_sheets = [str(x['properties']['title']) for x in data['sheets']]
             data_source.sheet = available_sheets[0]
 
         # https://developers.google.com/sheets/api/samples/reading
         read_sheet_endpoint = f'{data_source.spreadsheet_id}/values/{data_source.sheet}'
-        full_url = f'{self.baseroute}{read_sheet_endpoint}'
+        full_url = f'{self._baseroute}{read_sheet_endpoint}'
 
         data = self._run_fetch(full_url, access_token)['values']
         df = pd.DataFrame(data)
@@ -132,10 +160,11 @@ def get_status(self) -> ConnectorStatus:
 
         If successful, returns a message with the email of the connected user account.
         """
-        if not self.secrets or 'access_token' not in self.secrets:
+        secrets = self.hidden_properties.get_hidden_properties().get('secrets')
+        if not secrets or 'access_token' not in secrets:
             return ConnectorStatus(status=False, error='Credentials are missing')
 
-        access_token = self.secrets['access_token']
+        access_token = secrets['access_token']
         try:
             user_info = self._run_fetch(
                 'https://www.googleapis.com/oauth2/v2/userinfo?alt=json', access_token

toucan_connectors/toucan_connector.py

@@ -204,8 +204,6 @@ def __init_subclass__(cls):
             raise TypeError(f'{cls.__name__} has no {e} attribute.')
         if 'bearer_integration' in cls.__fields__:
             cls.bearer_integration = cls.__fields__['bearer_integration'].default
-        if 'auth_flow' in cls.__fields__:
-            cls.auth_flow = cls.__fields__['auth_flow'].default
 
     def bearer_oauth_get_endpoint(
         self,