Skip to content

Commit

Permalink
feat: support to get resources from other projects (#84)
Browse files Browse the repository at this point in the history
  • Loading branch information
@portellaa
portellaa authored Jan 17, 2024
1 parent fadeed9 commit 7354ce2
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 27 deletions.
43 changes: 29 additions & 14 deletions src/ydata/sdk/connectors/connector.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from ydata.sdk.common.config import LOG_LEVEL
from ydata.sdk.common.exceptions import CredentialTypeError, InvalidConnectorError
from ydata.sdk.common.logger import create_logger
from ydata.sdk.common.types import UID
from ydata.sdk.common.types import UID, Project
from ydata.sdk.connectors._models.connector import Connector as mConnector
from ydata.sdk.connectors._models.connector_list import ConnectorsList
from ydata.sdk.connectors._models.connector_type import ConnectorType
Expand All @@ -25,18 +25,23 @@ class Connector(ModelFactoryMixin):
connector_type (Union[ConnectorType, str]): Type of the connector to be created
credentials (dict): Connector credentials
name (Optional[str]): (optional) Connector name
project (Optional[Project]): (optional) Project name for this Connector
client (Client): (optional) Client to connect to the backend
Attributes:
uid (UID): UID fo the connector instance (creating internally)
type (ConnectorType): Type of the connector
"""

def __init__(self, connector_type: Union[ConnectorType, str] = None, credentials: Optional[Dict] = None, name: Optional[str] = None, client: Optional[Client] = None):
def __init__(
self, connector_type: Union[ConnectorType, str] = None, credentials: Optional[Dict] = None,
name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None):
self._init_common(client=client)
self._model: Optional[mConnector] = self._create_model(
connector_type, credentials, name, client=client)

self._project = project

@init_client
def _init_common(self, client: Optional[Client] = None):
self._client = client
Expand All @@ -52,20 +57,22 @@ def type(self) -> ConnectorType:

@staticmethod
@init_client
def get(uid: UID, client: Optional[Client] = None) -> "Connector":
def get(uid: UID, project: Optional[Project] = None, client: Optional[Client] = None) -> "Connector":
"""Get an existing connector.
Arguments:
uid (UID): Connector identifier
client (Client): (optional) Client to connect to the backend
project (Optional[Project]): (optional) Project name from where to get the connector
client (Optional[Client]): (optional) Client to connect to the backend
Returns:
Connector
"""
connectors: ConnectorsList = Connector.list(client=client)
data = connectors.get_by_uid(uid)
model = mConnector(**data)
connector = ModelFactoryMixin._init_from_model_data(Connector, model)
response = client.get(f'/connector/{uid}', project=project)
data = response.json()
connector = Connector._init_from_model_data(Connector, mConnector(**data))
connector._project = project

return connector

@staticmethod
Expand Down Expand Up @@ -104,27 +111,34 @@ def _init_credentials(connector_type: ConnectorType, credentials: Union[str, Pat
return _credentials

@staticmethod
def create(connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], name: Optional[str] = None, client: Optional[Client] = None) -> "Connector":
def create(
connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials],
name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None
) -> "Connector":
"""Create a new connector.
Arguments:
connector_type (Union[ConnectorType, str]): Type of the connector to be created
credentials (dict): Connector credentials
name (Optional[str]): (optional) Connector name
project (Optional[Project]): (optional) Project where to create the connector
client (Client): (optional) Client to connect to the backend
Returns:
New connector
"""
model = Connector._create_model(
connector_type=connector_type, credentials=credentials, name=name, client=client)
connector_type=connector_type, credentials=credentials, name=name, project=project, client=client)
connector = ModelFactoryMixin._init_from_model_data(
Connector, model)
connector._project = project
return connector

@classmethod
@init_client
def _create_model(cls, connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], name: Optional[str] = None, client: Optional[Client] = None) -> mConnector:
def _create_model(
cls, connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials],
name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None) -> mConnector:
_name = name if name is not None else str(uuid4())
_connector_type = Connector._init_connector_type(connector_type)
_credentials = Connector._init_credentials(_connector_type, credentials)
Expand All @@ -133,23 +147,24 @@ def _create_model(cls, connector_type: Union[ConnectorType, str], credentials: U
"credentials": _credentials.dict(by_alias=True),
"name": _name
}
response = client.post('/connector/', json=payload)
response = client.post('/connector/', project=project, json=payload)
data: list = response.json()

return mConnector(**data)

@staticmethod
@init_client
def list(client: Optional[Client] = None) -> ConnectorsList:
def list(project: Optional[Project] = None, client: Optional[Client] = None) -> ConnectorsList:
"""List the connectors instances.
Arguments:
project (Optional[Project]): (optional) Project name from where to list the connectors
client (Client): (optional) Client to connect to the backend
Returns:
List of connectors
"""
response = client.get('/connector')
response = client.get('/connector', project=project)
data: list = response.json()
return ConnectorsList(data)

Expand Down
55 changes: 42 additions & 13 deletions src/ydata/sdk/datasources/datasource.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from ydata.sdk.common.client.utils import init_client
from ydata.sdk.common.config import BACKOFF, LOG_LEVEL
from ydata.sdk.common.logger import create_logger
from ydata.sdk.common.types import UID
from ydata.sdk.common.types import UID, Project
from ydata.sdk.connectors.connector import Connector, ConnectorType
from ydata.sdk.datasources._models.connector_to_datasource import CONNECTOR_TO_DATASOURCE
from ydata.sdk.datasources._models.datasource import DataSource as mDataSource
Expand All @@ -26,6 +26,7 @@ class DataSource(ModelFactoryMixin):
connector (Connector): Connector from which the datasource is created
datatype (Optional[Union[DataSourceType, str]]): (optional) DataSource type
name (Optional[str]): (optional) DataSource name
project (Optional[Project]): (optional) Project name for this datasource
wait_for_metadata (bool): If `True`, wait until the metadata is fully calculated
client (Client): (optional) Client to connect to the backend
**config: Datasource specific configuration
Expand All @@ -37,15 +38,22 @@ class DataSource(ModelFactoryMixin):
metadata (Metadata): Metadata associated to the datasource
"""

def __init__(self, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, name: Optional[str] = None, wait_for_metadata: bool = True, client: Optional[Client] = None, **config):
def __init__(
self, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR,
name: Optional[str] = None, project: Optional[Project] = None, wait_for_metadata: bool = True,
client: Optional[Client] = None, **config
):
datasource_type = CONNECTOR_TO_DATASOURCE.get(connector.type)
self._init_common(client=client)
self._model: Optional[mDataSource] = self._create_model(
connector=connector, datasource_type=datasource_type, datatype=datatype, config=config, name=name, client=self._client)
connector=connector, datasource_type=datasource_type, datatype=datatype,
config=config, name=name, client=self._client)

if wait_for_metadata:
self._model = DataSource._wait_for_metadata(self)._model

self._project = project

@init_client
def _init_common(self, client: Optional[Client] = None):
self._client = client
Expand Down Expand Up @@ -73,11 +81,12 @@ def metadata(self) -> Metadata:

@staticmethod
@init_client
def list(client: Optional[Client] = None) -> DataSourceList:
def list(project: Optional[Project] = None, client: Optional[Client] = None) -> DataSourceList:
"""List the [`DataSource`][ydata.sdk.datasources.DataSource]
instances.
Arguments:
project (Optional[Project]): (optional) Project name from where to list the datasources
client (Client): (optional) Client to connect to the backend
Returns:
Expand All @@ -90,40 +99,47 @@ def __process_data(data: list) -> list:
e.pop(k, None)
return data

response = client.get('/datasource')
response = client.get('/datasource', project=project)
data: list = response.json()
data = __process_data(data)

return DataSourceList(data)

@staticmethod
@init_client
def get(uid: UID, client: Optional[Client] = None) -> "DataSource":
def get(uid: UID, project: Optional[Project] = None, client: Optional[Client] = None) -> "DataSource":
"""Get an existing [`DataSource`][ydata.sdk.datasources.DataSource].
Arguments:
uid (UID): DataSource identifier
project (Optional[Project]): (optional) Project name from where to get the connector
client (Client): (optional) Client to connect to the backend
Returns:
DataSource
"""
response = client.get(f'/datasource/{uid}')
response = client.get(f'/datasource/{uid}', project=project)
data: list = response.json()
datasource_type = CONNECTOR_TO_DATASOURCE.get(
ConnectorType(data['connector']['type']))
model = DataSource._model_from_api(data, datasource_type)
datasource = ModelFactoryMixin._init_from_model_data(DataSource, model)
datasource._project = project
return datasource

@classmethod
def create(cls, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, name: Optional[str] = None, wait_for_metadata: bool = True, client: Optional[Client] = None, **config) -> "DataSource":
def create(
cls, connector: Connector, datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR,
name: Optional[str] = None, project: Optional[Project] = None, wait_for_metadata: bool = True,
client: Optional[Client] = None, **config
) -> "DataSource":
"""Create a new [`DataSource`][ydata.sdk.datasources.DataSource].
Arguments:
connector (Connector): Connector from which the datasource is created
datatype (Optional[Union[DataSourceType, str]]): (optional) DataSource type
name (Optional[str]): (optional) DataSource name
project (Optional[Project]): (optional) Project name for this datasource
wait_for_metadata (bool): If `True`, wait until the metadata is fully calculated
client (Client): (optional) Client to connect to the backend
**config: Datasource specific configuration
Expand All @@ -132,22 +148,35 @@ def create(cls, connector: Connector, datatype: Optional[Union[DataSourceType, s
DataSource
"""
datasource_type = CONNECTOR_TO_DATASOURCE.get(connector.type)
return cls._create(connector=connector, datasource_type=datasource_type, datatype=datatype, config=config, name=name, wait_for_metadata=wait_for_metadata, client=client)
return cls._create(
connector=connector, datasource_type=datasource_type, datatype=datatype, config=config, name=name,
project=project, wait_for_metadata=wait_for_metadata, client=client)

@classmethod
def _create(cls, connector: Connector, datasource_type: Type[mDataSource], datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None, name: Optional[str] = None, wait_for_metadata: bool = True, client: Optional[Client] = None) -> "DataSource":
def _create(
cls, connector: Connector, datasource_type: Type[mDataSource],
datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None,
name: Optional[str] = None, project: Optional[Project] = None, wait_for_metadata: bool = True,
client: Optional[Client] = None
) -> "DataSource":
model = DataSource._create_model(
connector, datasource_type, datatype, config, name, client)
connector, datasource_type, datatype, config, name, project, client)
datasource = ModelFactoryMixin._init_from_model_data(DataSource, model)

if wait_for_metadata:
datasource._model = DataSource._wait_for_metadata(datasource)._model

datasource._project = project

return datasource

@classmethod
@init_client
def _create_model(cls, connector: Connector, datasource_type: Type[mDataSource], datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None, name: Optional[str] = None, client: Optional[Client] = None) -> mDataSource:
def _create_model(
cls, connector: Connector, datasource_type: Type[mDataSource],
datatype: Optional[Union[DataSourceType, str]] = DataSourceType.TABULAR, config: Optional[Dict] = None,
name: Optional[str] = None, project: Optional[Project] = None, client: Optional[Client] = None
) -> mDataSource:
_name = name if name is not None else str(uuid4())
_config = config if config is not None else {}
payload = {
Expand All @@ -161,7 +190,7 @@ def _create_model(cls, connector: Connector, datasource_type: Type[mDataSource],
if connector.type != ConnectorType.FILE:
_config = datasource_type(**config).to_payload()
payload.update(_config)
response = client.post('/datasource/', json=payload)
response = client.post('/datasource/', project=project, json=payload)
data: list = response.json()
return DataSource._model_from_api(data, datasource_type)

Expand Down

0 comments on commit 7354ce2

Please sign in to comment.