From ae36106ed91ff8a2f87e4a022dc2902bc5b323c1 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Mon, 25 Mar 2024 11:23:12 +0100 Subject: [PATCH 01/17] initial Python client --- clients/client-python/.gitignore | 163 +++++++++++++++++- .../__init__.py | 4 +- .../{__init__.py => gravitino/constants.py} | 4 +- .../core => gravitino}/dto.py | 2 + clients/client-python/gravitino/exceptions.py | 0 .../gravitino/gravitino_client.py | 41 +++++ clients/client-python/gravitino/service.py | 60 +++++++ clients/client-python/gravitino/typing.py | 8 + clients/client-python/gravitino/utils.py | 36 ++++ .../gravitino_client/core/__init__.py | 6 - .../gravitino_client/core/gravitino_client.py | 56 ------ clients/client-python/requirements.txt | 3 +- clients/client-python/setup.py | 39 +++++ clients/client-python/tests/__init__.py | 4 - .../tests/core/test_gravitino_client.py | 6 +- 15 files changed, 355 insertions(+), 77 deletions(-) rename clients/client-python/{gravitino_client => gravitino}/__init__.py (62%) rename clients/client-python/{__init__.py => gravitino/constants.py} (84%) rename clients/client-python/{gravitino_client/core => gravitino}/dto.py (99%) create mode 100644 clients/client-python/gravitino/exceptions.py create mode 100644 clients/client-python/gravitino/gravitino_client.py create mode 100644 clients/client-python/gravitino/service.py create mode 100644 clients/client-python/gravitino/typing.py create mode 100644 clients/client-python/gravitino/utils.py delete mode 100644 clients/client-python/gravitino_client/core/__init__.py delete mode 100644 clients/client-python/gravitino_client/core/gravitino_client.py create mode 100644 clients/client-python/setup.py delete mode 100644 clients/client-python/tests/__init__.py diff --git a/clients/client-python/.gitignore b/clients/client-python/.gitignore index e0549b75b57..6769e21d99a 100644 --- a/clients/client-python/.gitignore +++ b/clients/client-python/.gitignore @@ -1,5 +1,160 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. - +# Byte-compiled / optimized / DLL files __pycache__/ -.pytest_cache/ \ No newline at end of file +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/clients/client-python/gravitino_client/__init__.py b/clients/client-python/gravitino/__init__.py similarity index 62% rename from clients/client-python/gravitino_client/__init__.py rename to clients/client-python/gravitino/__init__.py index c064f0b0f0c..27cd5b7cc8e 100644 --- a/clients/client-python/gravitino_client/__init__.py +++ b/clients/client-python/gravitino/__init__.py @@ -1,4 +1,6 @@ """ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. -""" \ No newline at end of file +""" + +from gravitino.gravitino_client import GravitinoClient diff --git a/clients/client-python/__init__.py b/clients/client-python/gravitino/constants.py similarity index 84% rename from clients/client-python/__init__.py rename to clients/client-python/gravitino/constants.py index c064f0b0f0c..bca1d178c49 100644 --- a/clients/client-python/__init__.py +++ b/clients/client-python/gravitino/constants.py @@ -1,4 +1,6 @@ """ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. -""" \ No newline at end of file +""" + +TIMEOUT = 10 diff --git a/clients/client-python/gravitino_client/core/dto.py b/clients/client-python/gravitino/dto.py similarity index 99% rename from clients/client-python/gravitino_client/core/dto.py rename to clients/client-python/gravitino/dto.py index f6fc0f52c44..eb262ae4bfb 100644 --- a/clients/client-python/gravitino_client/core/dto.py +++ b/clients/client-python/gravitino/dto.py @@ -2,8 +2,10 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from dataclasses import dataclass + @dataclass class VersionDTO: version: str diff --git a/clients/client-python/gravitino/exceptions.py b/clients/client-python/gravitino/exceptions.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/clients/client-python/gravitino/gravitino_client.py b/clients/client-python/gravitino/gravitino_client.py new file mode 100644 index 00000000000..b9c45e6d5a3 --- /dev/null +++ b/clients/client-python/gravitino/gravitino_client.py @@ -0,0 +1,41 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" + +import re + +from gravitino.constants import TIMEOUT +from gravitino.service import Service + + +class GravitinoClient: + def __init__( + self, + host: str, + port: int = 8090, + prefix: str = "/api", + timeout: int = TIMEOUT, + debug: bool = False, + ) -> None: + if re.search(r"^https?:\/\/", host): + _host = host.rstrip("/") + else: + _host = f"http://{host.rstrip('/')}" + + if not re.search(r"[0-9]{2,5}$", _host): + _host = f"{_host}:{port}" + + _base_url = f"{_host}/{prefix.strip('/')}" + self.service = Service(_base_url, timeout) + self.debug = debug + + @property + def version(self): + return self.service.get_version() + + def __getattr__(self, metalake): + return self.service.get_metalake(metalake) + + def __dir__(self): + return ['the_first_metalake', 'metalake_demo'] diff --git a/clients/client-python/gravitino/service.py b/clients/client-python/gravitino/service.py new file mode 100644 index 00000000000..193e3ad0f9f --- /dev/null +++ b/clients/client-python/gravitino/service.py @@ -0,0 +1,60 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" + +from gravitino.utils import HTTPClient +from gravitino.typing import JSON_ro +from gravitino.constants import TIMEOUT + + +class Service: + def __init__( + self, + url: str, + timeout: int = TIMEOUT, + ) -> None: + self.base_url = url + self.http_client = HTTPClient(timeout=timeout) + + def get_version(self) -> JSON_ro: + return self.http_client.get(f"{self.base_url}/version") + + def get_metalakes(self) -> JSON_ro: + return self.http_client.get(f"{self.base_url}/metalakes") + + def get_metalake(self, metalake: str) -> JSON_ro: + return self.http_client.get(f"{self.base_url}/metalakes/{metalake}") + + def get_catalogs(self, metalake: str) -> JSON_ro: + return self.http_client.get(f"{self.base_url}/metalakes/{metalake}/catalogs/") + + def get_catalog(self, metalake: str, catalog: str) -> JSON_ro: + return self.http_client.get( + f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}" + ) + + def get_schemas(self, metalake: str, catalog: str) -> JSON_ro: + return self.http_client.get( + f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas" + ) + + def get_schema(self, metalake: str, catalog: str, schema: str) -> JSON_ro: + return self.http_client.get( + f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}" + ) + + def get_tables(self, metalake: str, catalog: str, schema: str) -> JSON_ro: + return self.http_client.get( + f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables" + ) + + def get_table(self, metalake: str, catalog: str, schema: str, table: str) -> JSON_ro: + return self.http_client.get( + f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}" + ) + + def get_partitions(self, metalake: str, catalog: str, schema: str, table: str) -> JSON_ro: + return self.http_client.get( + f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}/partitions" + ) diff --git a/clients/client-python/gravitino/typing.py b/clients/client-python/gravitino/typing.py new file mode 100644 index 00000000000..717b1f700ad --- /dev/null +++ b/clients/client-python/gravitino/typing.py @@ -0,0 +1,8 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" +from typing import Mapping, Sequence, Union, TypeAlias + +# https://github.com/python/typing/issues/182#issuecomment-1320974824 +JSON_ro: TypeAlias = Union[Mapping[str, "JSON_ro"], Sequence["JSON_ro"], str, int, float, bool, None] diff --git a/clients/client-python/gravitino/utils.py b/clients/client-python/gravitino/utils.py new file mode 100644 index 00000000000..e3dd5f19a50 --- /dev/null +++ b/clients/client-python/gravitino/utils.py @@ -0,0 +1,36 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" + +from requests import request +from requests.exceptions import HTTPError + +from gravitino.constants import TIMEOUT + + +class HTTPClient: + def __init__(self, timeout=TIMEOUT): + pass + + def _request( + self, method, endpoint, params=None, json=None, headers=None, timeout=None + ): + try: + response = request(method, endpoint) + response.raise_for_status() + return response.json() + except HTTPError as e: + raise HTTPError(f"Failed to retrieve version information: {e}") + + def get(self, endpoint, params=None, **kwargs): + return self._request("get", endpoint, params=params, **kwargs) + + def delete(self, endpoint, **kwargs): + return self._request("delete", endpoint, **kwargs) + + def post(self, endpoint, json=None, **kwargs): + return self._request("post", endpoint, json=json, **kwargs) + + def put(self, endpoint, json=None, **kwargs): + return self._request("put", endpoint, json=json, **kwargs) diff --git a/clients/client-python/gravitino_client/core/__init__.py b/clients/client-python/gravitino_client/core/__init__.py deleted file mode 100644 index 16d5ed38fd3..00000000000 --- a/clients/client-python/gravitino_client/core/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" -from .gravitino_client import GravitinoClient -from .dto import VersionDTO \ No newline at end of file diff --git a/clients/client-python/gravitino_client/core/gravitino_client.py b/clients/client-python/gravitino_client/core/gravitino_client.py deleted file mode 100644 index eed39431b0e..00000000000 --- a/clients/client-python/gravitino_client/core/gravitino_client.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" -import requests -from requests.exceptions import HTTPError -from gravitino_client.core.dto import VersionDTO - - -class GravitinoClient: - """ - Gravitino Client for interacting with the Gravitino API, allowing the client to list, load, - create, and alter Metalakes. - - Attributes: - base_url (str): The base URL of the Gravitino API to which the client will make requests. - - Args: - base_url (str): The base URL for the Gravitino API. - """ - - def __init__(self, base_url): - """ - Initializes a new instance of the GravitinoClient. - - Args: - base_url (str): The base URL for the Gravitino API where the client will send requests. - """ - self.base_url = base_url - - def getVersion(self) -> VersionDTO: - """ - Retrieves the version information from the Gravitino API. - - This method makes a GET request to the Gravitino API and extracts the version information from the response, - wrapping it into a VersionDTO object. - - Returns: - VersionDTO: An object containing the version details, including version, compile date, and git commit hash. - - Raises: - HTTPError: An error from the requests library if the HTTP request returned an unsuccessful status code. - """ - try: - response = requests.get(f"{self.base_url}/api/version") - response.raise_for_status() - version_data = response.json() - version_info = version_data.get("version") - - return VersionDTO( - version=version_info['version'], - compile_date=version_info['compileDate'], - git_commit=version_info['gitCommit'] - ) - except HTTPError as e: - raise HTTPError(f"Failed to retrieve version information: {e}") diff --git a/clients/client-python/requirements.txt b/clients/client-python/requirements.txt index 5f118436582..6cf7e8bda22 100644 --- a/clients/client-python/requirements.txt +++ b/clients/client-python/requirements.txt @@ -1,4 +1,3 @@ # Copyright 2024 Datastrato Pvt Ltd. # This software is licensed under the Apache License version 2. -pytest~=8.0.1 -requests~=2.31.0 \ No newline at end of file +requests~=2.31.0 diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py new file mode 100644 index 00000000000..ff2bcfedb39 --- /dev/null +++ b/clients/client-python/setup.py @@ -0,0 +1,39 @@ +import io +import os +from setuptools import find_packages, setup + + +def read(*paths, **kwargs): + content = "" + with io.open( + os.path.join(os.path.dirname(__file__), *paths), + encoding=kwargs.get("encoding", "utf8"), + ) as open_file: + content = open_file.read().strip() + return content + + +def read_requirements(path): + return [ + line.strip() + for line in read(path).split("\n") + if not line.startswith(('"', "#", "-", "git+")) + ] + + +setup( + name="gravitino", + version="0.0.1", + description="project description TBD", + url="https://github.com/datastrato/gravitino", + long_description=read("README.md"), + long_description_content_type="text/markdown", + author="datastrato", + packages=find_packages(include=["gravitino", ".*"]), + install_requires=read_requirements("requirements.txt"), + extras_require={ + "dev": [ + "pytest~=8.0.1", + ] + }, +) diff --git a/clients/client-python/tests/__init__.py b/clients/client-python/tests/__init__.py deleted file mode 100644 index c064f0b0f0c..00000000000 --- a/clients/client-python/tests/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" \ No newline at end of file diff --git a/clients/client-python/tests/core/test_gravitino_client.py b/clients/client-python/tests/core/test_gravitino_client.py index 7d5ffba9eca..742f507e6ea 100644 --- a/clients/client-python/tests/core/test_gravitino_client.py +++ b/clients/client-python/tests/core/test_gravitino_client.py @@ -5,7 +5,7 @@ import pytest import requests from unittest.mock import MagicMock -from gravitino_client.core import GravitinoClient, VersionDTO +from gravitino import Service, VersionDTO @pytest.fixture @@ -30,7 +30,7 @@ def test_get_version_success(mock_get): } } - client = GravitinoClient(base_url="http://localhost:8090") + client = Service(base_url="http://localhost:8090") version_data = client.getVersion() assert version_data == expected_version_dto @@ -39,7 +39,7 @@ def test_get_version_success(mock_get): def test_get_version_http_error(mock_get): mock_get.side_effect = requests.exceptions.HTTPError - client = GravitinoClient(base_url="http://localhost:8090") + client = Service(base_url="http://localhost:8090") with pytest.raises(requests.exceptions.HTTPError): client.getVersion() From 1c916da51b560d62b08780a7c197f7bb46d2236b Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Mon, 25 Mar 2024 21:29:54 +0100 Subject: [PATCH 02/17] wip --- .../gravitino/gravitino_client.py | 35 ++++++++++++++++--- clients/client-python/gravitino/service.py | 12 ++++++- clients/client-python/gravitino/typing.py | 4 +-- clients/client-python/gravitino/utils.py | 13 +++++++ clients/client-python/tests/test_foobar.py | 7 ++++ 5 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 clients/client-python/tests/test_foobar.py diff --git a/clients/client-python/gravitino/gravitino_client.py b/clients/client-python/gravitino/gravitino_client.py index b9c45e6d5a3..d81ba4c66bd 100644 --- a/clients/client-python/gravitino/gravitino_client.py +++ b/clients/client-python/gravitino/gravitino_client.py @@ -9,10 +9,31 @@ from gravitino.service import Service +class MetaLake: + pass + + +class Catalog: + pass + + +class Schema: + pass + + +class Table: + pass + + +class Partition: + pass + + class GravitinoClient: def __init__( self, host: str, + protocol: str = 'http', port: int = 8090, prefix: str = "/api", timeout: int = TIMEOUT, @@ -21,7 +42,7 @@ def __init__( if re.search(r"^https?:\/\/", host): _host = host.rstrip("/") else: - _host = f"http://{host.rstrip('/')}" + _host = f"{protocol}://{host.rstrip('/')}" if not re.search(r"[0-9]{2,5}$", _host): _host = f"{_host}:{port}" @@ -34,8 +55,14 @@ def __init__( def version(self): return self.service.get_version() - def __getattr__(self, metalake): + def get_metalakes(self): + return self.service.get_metalakes() + + def get_metalake(self, metalake: str): return self.service.get_metalake(metalake) - def __dir__(self): - return ['the_first_metalake', 'metalake_demo'] + # def __getattr__(self, metalake): + # return self.service.get_metalake(metalake) + # + # def __dir__(self): + # return ['the_first_metalake', 'metalake_demo'] diff --git a/clients/client-python/gravitino/service.py b/clients/client-python/gravitino/service.py index 193e3ad0f9f..6067dabfe90 100644 --- a/clients/client-python/gravitino/service.py +++ b/clients/client-python/gravitino/service.py @@ -3,7 +3,7 @@ This software is licensed under the Apache License version 2. """ -from gravitino.utils import HTTPClient +from gravitino.utils import HTTPClient, unpack from gravitino.typing import JSON_ro from gravitino.constants import TIMEOUT @@ -17,43 +17,53 @@ def __init__( self.base_url = url self.http_client = HTTPClient(timeout=timeout) + @unpack("version") def get_version(self) -> JSON_ro: return self.http_client.get(f"{self.base_url}/version") + @unpack("metalakes") def get_metalakes(self) -> JSON_ro: return self.http_client.get(f"{self.base_url}/metalakes") + @unpack("metalake") def get_metalake(self, metalake: str) -> JSON_ro: return self.http_client.get(f"{self.base_url}/metalakes/{metalake}") + @unpack("identifiers") def get_catalogs(self, metalake: str) -> JSON_ro: return self.http_client.get(f"{self.base_url}/metalakes/{metalake}/catalogs/") + @unpack("catalog") def get_catalog(self, metalake: str, catalog: str) -> JSON_ro: return self.http_client.get( f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}" ) + @unpack("identifiers") def get_schemas(self, metalake: str, catalog: str) -> JSON_ro: return self.http_client.get( f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas" ) + @unpack("schema") def get_schema(self, metalake: str, catalog: str, schema: str) -> JSON_ro: return self.http_client.get( f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}" ) + @unpack("identifiers") def get_tables(self, metalake: str, catalog: str, schema: str) -> JSON_ro: return self.http_client.get( f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables" ) + @unpack("table") def get_table(self, metalake: str, catalog: str, schema: str, table: str) -> JSON_ro: return self.http_client.get( f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}" ) + @unpack("names") def get_partitions(self, metalake: str, catalog: str, schema: str, table: str) -> JSON_ro: return self.http_client.get( f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}/partitions" diff --git a/clients/client-python/gravitino/typing.py b/clients/client-python/gravitino/typing.py index 717b1f700ad..3b74b65bdf8 100644 --- a/clients/client-python/gravitino/typing.py +++ b/clients/client-python/gravitino/typing.py @@ -2,7 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ -from typing import Mapping, Sequence, Union, TypeAlias +from typing import Mapping, Sequence, Union # https://github.com/python/typing/issues/182#issuecomment-1320974824 -JSON_ro: TypeAlias = Union[Mapping[str, "JSON_ro"], Sequence["JSON_ro"], str, int, float, bool, None] +JSON_ro = Union[Mapping[str, "JSON_ro"], Sequence["JSON_ro"], str, int, float, bool, None] diff --git a/clients/client-python/gravitino/utils.py b/clients/client-python/gravitino/utils.py index e3dd5f19a50..15e0c013ada 100644 --- a/clients/client-python/gravitino/utils.py +++ b/clients/client-python/gravitino/utils.py @@ -34,3 +34,16 @@ def post(self, endpoint, json=None, **kwargs): def put(self, endpoint, json=None, **kwargs): return self._request("put", endpoint, json=json, **kwargs) + + +def unpack(path: str): + def decorator(func): + def wrapper(*args, **kwargs): + rv = func(*args, **kwargs) + for p in path.split('.'): + if p not in rv: + raise KeyError(f"The path '{path}' can't find in dict") + rv = rv.get(p) + return rv + return wrapper + return decorator diff --git a/clients/client-python/tests/test_foobar.py b/clients/client-python/tests/test_foobar.py new file mode 100644 index 00000000000..c4faf3f8f11 --- /dev/null +++ b/clients/client-python/tests/test_foobar.py @@ -0,0 +1,7 @@ +from gravitino import GravitinoClient + + +# def test_foobar(): +gravitino = GravitinoClient("http://localhost") +# print(gravitino.service.get_partitions("metalake_demo", "catalog_hive", "sales", "customers")) +print(gravitino.get_metalakes()) From ba4abfb55c58e4564087db05a6b5c1cbe008b160 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Mon, 25 Mar 2024 23:08:17 +0100 Subject: [PATCH 03/17] wip --- .../gravitino/gravitino_client.py | 85 +++++++++++++++---- clients/client-python/gravitino/service.py | 17 +++- clients/client-python/tests/test_foobar.py | 7 -- 3 files changed, 80 insertions(+), 29 deletions(-) delete mode 100644 clients/client-python/tests/test_foobar.py diff --git a/clients/client-python/gravitino/gravitino_client.py b/clients/client-python/gravitino/gravitino_client.py index d81ba4c66bd..0efb90aeefc 100644 --- a/clients/client-python/gravitino/gravitino_client.py +++ b/clients/client-python/gravitino/gravitino_client.py @@ -6,27 +6,81 @@ import re from gravitino.constants import TIMEOUT -from gravitino.service import Service +from gravitino.service import initialize_service, service +from gravitino.typing import JSON_ro class MetaLake: - pass + def __init__(self, payload: JSON_ro): + self.name = payload.get("name") + self.payload = payload + self.service = service["service"] + self.catalogs = self.service.get_catalogs(self.name) + + def __repr__(self): + return f"MetaLake<{self.name}>" + + def __getattr__(self, catalog_name): + if catalog_name in dir(self): + return Catalog(self.name, catalog_name) + + def __dir__(self): + return [catalog["name"] for catalog in self.catalogs] class Catalog: - pass + def __init__(self, metalake_name: str, catalog_name: str): + self.metalake_name = metalake_name + self.catalog_name = catalog_name + self.name = catalog_name + self.service = service["service"] + self.schemas = self.service.get_schemas(metalake_name, catalog_name) + + def __repr__(self): + return f"Catalog<{self.name}>" + + def __getattr__(self, schema_name): + if schema_name in dir(self): + return Schema(self.metalake_name, self.catalog_name, schema_name) + + def __dir__(self): + return [schema["name"] for schema in self.schemas] class Schema: - pass + def __init__(self, metalake_name: str, catalog_name: str, schema_name: str): + self.metalake_name = metalake_name + self.catalog_name = catalog_name + self.schema_name = schema_name + self.name = schema_name + self.service = service["service"] + self.tables = self.service.get_tables(metalake_name, catalog_name, schema_name) + + def __repr__(self): + return f"Schema<{self.name}>" + + def __getattr__(self, table_name): + if table_name in dir(self): + return Table(self.metalake_name, self.catalog_name, self.schema_name, table_name) + + def __dir__(self): + return [table["name"] for table in self.tables] class Table: - pass + def __init__(self, metalake_name: str, catalog_name: str, schema_name: str, table_name: str): + self.metalake_name = metalake_name + self.catalog_name = catalog_name + self.schema_name = schema_name + self.table_name = table_name + self.name = schema_name + self.service = service["service"] + def __repr__(self): + return f"Table<{self.name}>" -class Partition: - pass + def info(self): + return self.service.get_table(self.metalake_name, self.catalog_name, self.schema_name, self.table_name) class GravitinoClient: @@ -48,21 +102,16 @@ def __init__( _host = f"{_host}:{port}" _base_url = f"{_host}/{prefix.strip('/')}" - self.service = Service(_base_url, timeout) + initialize_service(_base_url, timeout) + self.service = service["service"] self.debug = debug @property def version(self): return self.service.get_version() - def get_metalakes(self): - return self.service.get_metalakes() - - def get_metalake(self, metalake: str): - return self.service.get_metalake(metalake) + def get_metalakes(self) -> [MetaLake]: + return [MetaLake(metalake) for metalake in self.service.get_metalakes()] - # def __getattr__(self, metalake): - # return self.service.get_metalake(metalake) - # - # def __dir__(self): - # return ['the_first_metalake', 'metalake_demo'] + def get_metalake(self, metalake: str) -> MetaLake: + return MetaLake(self.service.get_metalake(metalake)) diff --git a/clients/client-python/gravitino/service.py b/clients/client-python/gravitino/service.py index 6067dabfe90..385b69044c1 100644 --- a/clients/client-python/gravitino/service.py +++ b/clients/client-python/gravitino/service.py @@ -8,11 +8,11 @@ from gravitino.constants import TIMEOUT -class Service: +class _Service: def __init__( - self, - url: str, - timeout: int = TIMEOUT, + self, + url: str, + timeout: int = TIMEOUT, ) -> None: self.base_url = url self.http_client = HTTPClient(timeout=timeout) @@ -68,3 +68,12 @@ def get_partitions(self, metalake: str, catalog: str, schema: str, table: str) - return self.http_client.get( f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}/partitions" ) + + +service = {} + + +def initialize_service(url: str, timeout: int = TIMEOUT): + global service + if not service: + service['service'] = _Service(url, timeout) diff --git a/clients/client-python/tests/test_foobar.py b/clients/client-python/tests/test_foobar.py deleted file mode 100644 index c4faf3f8f11..00000000000 --- a/clients/client-python/tests/test_foobar.py +++ /dev/null @@ -1,7 +0,0 @@ -from gravitino import GravitinoClient - - -# def test_foobar(): -gravitino = GravitinoClient("http://localhost") -# print(gravitino.service.get_partitions("metalake_demo", "catalog_hive", "sales", "customers")) -print(gravitino.get_metalakes()) From 0be1a6ea9b001bd1482ca07a6754f2c79b34f17b Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Mon, 25 Mar 2024 23:09:35 +0100 Subject: [PATCH 04/17] wip --- clients/client-python/gravitino/dto.py | 13 ------------- clients/client-python/gravitino/exceptions.py | 4 ++++ 2 files changed, 4 insertions(+), 13 deletions(-) delete mode 100644 clients/client-python/gravitino/dto.py diff --git a/clients/client-python/gravitino/dto.py b/clients/client-python/gravitino/dto.py deleted file mode 100644 index eb262ae4bfb..00000000000 --- a/clients/client-python/gravitino/dto.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -from dataclasses import dataclass - - -@dataclass -class VersionDTO: - version: str - compile_date: str - git_commit: str diff --git a/clients/client-python/gravitino/exceptions.py b/clients/client-python/gravitino/exceptions.py index e69de29bb2d..5779a3ad252 100644 --- a/clients/client-python/gravitino/exceptions.py +++ b/clients/client-python/gravitino/exceptions.py @@ -0,0 +1,4 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" From f31225e91eab7bbdb91010ac356b4f74231112ec Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Mon, 25 Mar 2024 23:40:45 +0100 Subject: [PATCH 05/17] wip --- clients/client-python/tests/core/__init__.py | 4 -- .../tests/core/test_gravitino_client.py | 45 ------------------- 2 files changed, 49 deletions(-) delete mode 100644 clients/client-python/tests/core/__init__.py delete mode 100644 clients/client-python/tests/core/test_gravitino_client.py diff --git a/clients/client-python/tests/core/__init__.py b/clients/client-python/tests/core/__init__.py deleted file mode 100644 index c064f0b0f0c..00000000000 --- a/clients/client-python/tests/core/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" \ No newline at end of file diff --git a/clients/client-python/tests/core/test_gravitino_client.py b/clients/client-python/tests/core/test_gravitino_client.py deleted file mode 100644 index 742f507e6ea..00000000000 --- a/clients/client-python/tests/core/test_gravitino_client.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" -import pytest -import requests -from unittest.mock import MagicMock -from gravitino import Service, VersionDTO - - -@pytest.fixture -def mock_get(monkeypatch): - mock = MagicMock() - monkeypatch.setattr("requests.get", mock) - return mock - - -def test_get_version_success(mock_get): - expected_version_dto = VersionDTO( - version="0.3.2-SNAPSHOT", - compile_date="25/01/2024 00:04:59", - git_commit="cb7a604bf19b6f992f00529e938cdd1d37af0187" - ) - mock_get.return_value.json.return_value = { - "code": 0, - "version": { - "version": "0.3.2-SNAPSHOT", - "compileDate": "25/01/2024 00:04:59", - "gitCommit": "cb7a604bf19b6f992f00529e938cdd1d37af0187" - } - } - - client = Service(base_url="http://localhost:8090") - version_data = client.getVersion() - - assert version_data == expected_version_dto - - -def test_get_version_http_error(mock_get): - mock_get.side_effect = requests.exceptions.HTTPError - - client = Service(base_url="http://localhost:8090") - - with pytest.raises(requests.exceptions.HTTPError): - client.getVersion() From 11665e873e0724d797b11dfbfeb17c45e6b7a71d Mon Sep 17 00:00:00 2001 From: Shaofeng Shi Date: Wed, 27 Mar 2024 15:50:30 +0800 Subject: [PATCH 06/17] Update setup.py Add header for setup.py --- clients/client-python/setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py index ff2bcfedb39..61acc1ad24c 100644 --- a/clients/client-python/setup.py +++ b/clients/client-python/setup.py @@ -1,3 +1,7 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" import io import os from setuptools import find_packages, setup From 18db11d6af11cfd1675d3ce90fbfcecf0e5d8b13 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Wed, 27 Mar 2024 14:28:11 +0100 Subject: [PATCH 07/17] remove requests --- clients/client-python/README.md | 11 +- clients/client-python/gravitino/__init__.py | 2 +- .../gravitino/gravitino_client.py | 59 ++++++- clients/client-python/gravitino/service.py | 45 +++-- clients/client-python/gravitino/typing.py | 5 +- clients/client-python/gravitino/utils.py | 49 ------ .../client-python/gravitino/utils/__init__.py | 4 + .../gravitino/utils/exceptions.py | 95 +++++++++++ .../gravitino/utils/http_client.py | 158 ++++++++++++++++++ clients/client-python/requirements.txt | 3 - clients/client-python/setup.py | 15 +- 11 files changed, 351 insertions(+), 95 deletions(-) delete mode 100644 clients/client-python/gravitino/utils.py create mode 100644 clients/client-python/gravitino/utils/__init__.py create mode 100644 clients/client-python/gravitino/utils/exceptions.py create mode 100644 clients/client-python/gravitino/utils/http_client.py delete mode 100644 clients/client-python/requirements.txt diff --git a/clients/client-python/README.md b/clients/client-python/README.md index 8515e255469..d69ba3e14ef 100644 --- a/clients/client-python/README.md +++ b/clients/client-python/README.md @@ -5,9 +5,14 @@ # Quick Start -1. Install dependency +1. Install current library in your local machine. ```bash - pip install -r requirements.txt + pip install -e . ``` -2. After the packages is installed, you can simply run `pytest` in any directory inside gravitino/ \ No newline at end of file +# Development Environment + +1. Install dependency + ```bash + pip install -e .[dev] + ``` \ No newline at end of file diff --git a/clients/client-python/gravitino/__init__.py b/clients/client-python/gravitino/__init__.py index 27cd5b7cc8e..af49bcc12d9 100644 --- a/clients/client-python/gravitino/__init__.py +++ b/clients/client-python/gravitino/__init__.py @@ -3,4 +3,4 @@ This software is licensed under the Apache License version 2. """ -from gravitino.gravitino_client import GravitinoClient +from gravitino.gravitino_client import GravitinoClient, gravitino_metalake diff --git a/clients/client-python/gravitino/gravitino_client.py b/clients/client-python/gravitino/gravitino_client.py index 0efb90aeefc..c0deef555a9 100644 --- a/clients/client-python/gravitino/gravitino_client.py +++ b/clients/client-python/gravitino/gravitino_client.py @@ -61,14 +61,18 @@ def __repr__(self): def __getattr__(self, table_name): if table_name in dir(self): - return Table(self.metalake_name, self.catalog_name, self.schema_name, table_name) + return Table( + self.metalake_name, self.catalog_name, self.schema_name, table_name + ) def __dir__(self): return [table["name"] for table in self.tables] class Table: - def __init__(self, metalake_name: str, catalog_name: str, schema_name: str, table_name: str): + def __init__( + self, metalake_name: str, catalog_name: str, schema_name: str, table_name: str + ): self.metalake_name = metalake_name self.catalog_name = catalog_name self.schema_name = schema_name @@ -80,14 +84,17 @@ def __repr__(self): return f"Table<{self.name}>" def info(self): - return self.service.get_table(self.metalake_name, self.catalog_name, self.schema_name, self.table_name) + return self.service.get_table( + self.metalake_name, self.catalog_name, self.schema_name, self.table_name + ) class GravitinoClient: def __init__( self, host: str, - protocol: str = 'http', + *, + protocol: str = "http", port: int = 8090, prefix: str = "/api", timeout: int = TIMEOUT, @@ -106,6 +113,29 @@ def __init__( self.service = service["service"] self.debug = debug + @classmethod + def initialize_metalake( + cls, + host: str, + metalake_name: str, + *, + protocol: str = "http", + port: int = 8090, + prefix: str = "/api", + timeout: int = TIMEOUT, + debug: bool = False, + ) -> MetaLake: + # keep in mind, all constructors should include same interface as __init__ function + client = cls( + host, + protocol=protocol, + port=port, + prefix=prefix, + timeout=timeout, + debug=debug, + ) + return client.get_metalake(metalake_name) + @property def version(self): return self.service.get_version() @@ -115,3 +145,24 @@ def get_metalakes(self) -> [MetaLake]: def get_metalake(self, metalake: str) -> MetaLake: return MetaLake(self.service.get_metalake(metalake)) + + +def gravitino_metalake( + host: str, + metalake_name: str, + *, + protocol: str = "http", + port: int = 8090, + prefix: str = "/api", + timeout: int = TIMEOUT, + debug: bool = False, +) -> MetaLake: + return GravitinoClient.initialize_metalake( + host, + metalake_name, + protocol=protocol, + port=port, + prefix=prefix, + timeout=timeout, + debug=debug, + ) diff --git a/clients/client-python/gravitino/service.py b/clients/client-python/gravitino/service.py index 385b69044c1..59f8bba829d 100644 --- a/clients/client-python/gravitino/service.py +++ b/clients/client-python/gravitino/service.py @@ -3,70 +3,69 @@ This software is licensed under the Apache License version 2. """ -from gravitino.utils import HTTPClient, unpack +from gravitino.utils.http_client import HTTPClient, unpack from gravitino.typing import JSON_ro from gravitino.constants import TIMEOUT class _Service: def __init__( - self, - url: str, - timeout: int = TIMEOUT, + self, + url: str, + timeout: int = TIMEOUT, ) -> None: - self.base_url = url - self.http_client = HTTPClient(timeout=timeout) + self.http_client = HTTPClient(url, timeout=timeout) @unpack("version") def get_version(self) -> JSON_ro: - return self.http_client.get(f"{self.base_url}/version") + return self.http_client.get("/version") @unpack("metalakes") def get_metalakes(self) -> JSON_ro: - return self.http_client.get(f"{self.base_url}/metalakes") + return self.http_client.get("/metalakes") @unpack("metalake") def get_metalake(self, metalake: str) -> JSON_ro: - return self.http_client.get(f"{self.base_url}/metalakes/{metalake}") + return self.http_client.get(f"/metalakes/{metalake}") @unpack("identifiers") def get_catalogs(self, metalake: str) -> JSON_ro: - return self.http_client.get(f"{self.base_url}/metalakes/{metalake}/catalogs/") + return self.http_client.get(f"/metalakes/{metalake}/catalogs/") @unpack("catalog") def get_catalog(self, metalake: str, catalog: str) -> JSON_ro: - return self.http_client.get( - f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}" - ) + return self.http_client.get(f"/metalakes/{metalake}/catalogs/{catalog}") @unpack("identifiers") def get_schemas(self, metalake: str, catalog: str) -> JSON_ro: - return self.http_client.get( - f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas" - ) + return self.http_client.get(f"/metalakes/{metalake}/catalogs/{catalog}/schemas") @unpack("schema") def get_schema(self, metalake: str, catalog: str, schema: str) -> JSON_ro: return self.http_client.get( - f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}" + f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}" ) @unpack("identifiers") def get_tables(self, metalake: str, catalog: str, schema: str) -> JSON_ro: return self.http_client.get( - f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables" + f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables" ) @unpack("table") - def get_table(self, metalake: str, catalog: str, schema: str, table: str) -> JSON_ro: + def get_table( + self, metalake: str, catalog: str, schema: str, table: str + ) -> JSON_ro: return self.http_client.get( - f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}" + f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}" ) @unpack("names") - def get_partitions(self, metalake: str, catalog: str, schema: str, table: str) -> JSON_ro: + def get_partitions( + self, metalake: str, catalog: str, schema: str, table: str + ) -> JSON_ro: return self.http_client.get( - f"{self.base_url}/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}/partitions" + f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}/partitions" ) @@ -76,4 +75,4 @@ def get_partitions(self, metalake: str, catalog: str, schema: str, table: str) - def initialize_service(url: str, timeout: int = TIMEOUT): global service if not service: - service['service'] = _Service(url, timeout) + service["service"] = _Service(url, timeout) diff --git a/clients/client-python/gravitino/typing.py b/clients/client-python/gravitino/typing.py index 3b74b65bdf8..4dc7db893f4 100644 --- a/clients/client-python/gravitino/typing.py +++ b/clients/client-python/gravitino/typing.py @@ -2,7 +2,10 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from typing import Mapping, Sequence, Union # https://github.com/python/typing/issues/182#issuecomment-1320974824 -JSON_ro = Union[Mapping[str, "JSON_ro"], Sequence["JSON_ro"], str, int, float, bool, None] +JSON_ro = Union[ + Mapping[str, "JSON_ro"], Sequence["JSON_ro"], str, int, float, bool, None +] diff --git a/clients/client-python/gravitino/utils.py b/clients/client-python/gravitino/utils.py deleted file mode 100644 index 15e0c013ada..00000000000 --- a/clients/client-python/gravitino/utils.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -Copyright 2024 Datastrato Pvt Ltd. -This software is licensed under the Apache License version 2. -""" - -from requests import request -from requests.exceptions import HTTPError - -from gravitino.constants import TIMEOUT - - -class HTTPClient: - def __init__(self, timeout=TIMEOUT): - pass - - def _request( - self, method, endpoint, params=None, json=None, headers=None, timeout=None - ): - try: - response = request(method, endpoint) - response.raise_for_status() - return response.json() - except HTTPError as e: - raise HTTPError(f"Failed to retrieve version information: {e}") - - def get(self, endpoint, params=None, **kwargs): - return self._request("get", endpoint, params=params, **kwargs) - - def delete(self, endpoint, **kwargs): - return self._request("delete", endpoint, **kwargs) - - def post(self, endpoint, json=None, **kwargs): - return self._request("post", endpoint, json=json, **kwargs) - - def put(self, endpoint, json=None, **kwargs): - return self._request("put", endpoint, json=json, **kwargs) - - -def unpack(path: str): - def decorator(func): - def wrapper(*args, **kwargs): - rv = func(*args, **kwargs) - for p in path.split('.'): - if p not in rv: - raise KeyError(f"The path '{path}' can't find in dict") - rv = rv.get(p) - return rv - return wrapper - return decorator diff --git a/clients/client-python/gravitino/utils/__init__.py b/clients/client-python/gravitino/utils/__init__.py new file mode 100644 index 00000000000..5779a3ad252 --- /dev/null +++ b/clients/client-python/gravitino/utils/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" diff --git a/clients/client-python/gravitino/utils/exceptions.py b/clients/client-python/gravitino/utils/exceptions.py new file mode 100644 index 00000000000..147a1698362 --- /dev/null +++ b/clients/client-python/gravitino/utils/exceptions.py @@ -0,0 +1,95 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" + +import json + + +class HTTPError(Exception): + """Base of all other errors""" + + def __init__(self, error): + self.status_code = error.code + self.reason = error.reason + self.body = error.read() + self.headers = error.hdrs + + def json(self): + """ + :return: object of response error from the API + """ + try: + return json.loads(self.body.decode("utf-8")) + except json.decoder.JSONDecodeError: + return {"exception": self.body.decode("utf-8")} + + def __str__(self): + return self.json().get("exception") + + +class BadRequestsError(HTTPError): + pass + + +class UnauthorizedError(HTTPError): + pass + + +class ForbiddenError(HTTPError): + pass + + +class NotFoundError(HTTPError): + pass + + +class MethodNotAllowedError(HTTPError): + pass + + +class PayloadTooLargeError(HTTPError): + pass + + +class UnsupportedMediaTypeError(HTTPError): + pass + + +class TooManyRequestsError(HTTPError): + pass + + +class InternalServerError(HTTPError): + pass + + +class ServiceUnavailableError(HTTPError): + pass + + +class GatewayTimeoutError(HTTPError): + pass + + +err_dict = { + 400: BadRequestsError, + 401: UnauthorizedError, + 403: ForbiddenError, + 404: NotFoundError, + 405: MethodNotAllowedError, + 413: PayloadTooLargeError, + 415: UnsupportedMediaTypeError, + 429: TooManyRequestsError, + 500: InternalServerError, + 503: ServiceUnavailableError, + 504: GatewayTimeoutError, +} + + +def handle_error(error): + try: + exc = err_dict[error.code](error) + except KeyError: + return HTTPError(error) + return exc diff --git a/clients/client-python/gravitino/utils/http_client.py b/clients/client-python/gravitino/utils/http_client.py new file mode 100644 index 00000000000..e914babbbf4 --- /dev/null +++ b/clients/client-python/gravitino/utils/http_client.py @@ -0,0 +1,158 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" + +from urllib.request import Request, build_opener +from urllib.parse import urlencode +from urllib.error import HTTPError +import json as _json + +from gravitino.utils.exceptions import handle_error +from gravitino.constants import TIMEOUT + + +class Response: + def __init__(self, response): + self._status_code = response.getcode() + self._body = response.read() + self._headers = response.info() + self._url = response.url + + @property + def status_code(self): + return self._status_code + + @property + def url(self): + return self._url + + @property + def body(self): + return self._body + + @property + def headers(self): + return self._headers + + def json(self): + if self.body: + return _json.loads(self.body.decode("utf-8")) + else: + return None + + +class HTTPClient: + def __init__( + self, + host, + *, + request_headers=None, + timeout=TIMEOUT, + is_debug=False, + ) -> None: + self.host = host + self.request_headers = request_headers or {} + self.timeout = timeout + self.is_debug = is_debug + + def _build_url(self, endpoint=None, params=None): + url = self.host + + if endpoint: + url = "{}/{}".format(url.rstrip("/"), endpoint.lstrip("/")) + + if params: + params = {k: v for k, v in params.items() if v is not None} + url_values = urlencode(sorted(params.items()), True) + url = "{}?{}".format(url, url_values) + + return url + + def _update_headers(self, request_headers): + self.request_headers.update(request_headers) + + def _mask_auth_headers(self, headers): + if self.is_debug: + return headers + + _headers = {} + for key, value in headers.items(): + if key.lower() == "authorization": + _headers[key] = "******" + else: + _headers[key] = value + return _headers + + def _make_request(self, opener, request, timeout=None): + timeout = timeout or self.timeout + # logger.debug( + # """ + # ==========================[QUERY]=============================== + # method: {} + # url: {} + # headers: {} + # body: {} + # ==========================[QUERY]===============================""".format( # noqa + # request.get_method(), + # request.get_full_url(), + # self._mask_auth_headers(dict(request.header_items())), + # request.data, + # ) + # ) + try: + return opener.open(request, timeout=timeout) + except HTTPError as err: + exc = handle_error(err) + exc.__cause__ = None + raise exc + + def _request( + self, method, endpoint, params=None, json=None, headers=None, timeout=None + ): + method = method.upper() + request_data = None + + if headers: + self._update_headers(headers) + if json: + request_data = _json.dumps(json).encode("utf-8") + + opener = build_opener() + request = Request(self._build_url(endpoint, params), data=request_data) + if self.request_headers: + for key, value in self.request_headers.items(): + request.add_header(key, value) + if request_data and ("Content-Type" not in self.request_headers): + request.add_header("Content-Type", "application/json") + + request.get_method = lambda: method + return Response(self._make_request(opener, request, timeout=timeout)) + + def get(self, endpoint, params=None, **kwargs): + return self._request("get", endpoint, params=params, **kwargs) + + def delete(self, endpoint, **kwargs): + return self._request("delete", endpoint, **kwargs) + + def post(self, endpoint, json=None, **kwargs): + return self._request("post", endpoint, json=json, **kwargs) + + def put(self, endpoint, json=None, **kwargs): + return self._request("put", endpoint, json=json, **kwargs) + + +def unpack(path: str): + def decorator(func): + def wrapper(*args, **kwargs): + resp = func(*args, **kwargs) + rv = resp.json() + for p in path.split("."): + if p not in rv: + raise KeyError(f"The path '{path}' can't find in dict") + rv = rv.get(p) + return rv + + return wrapper + + return decorator diff --git a/clients/client-python/requirements.txt b/clients/client-python/requirements.txt deleted file mode 100644 index 6cf7e8bda22..00000000000 --- a/clients/client-python/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright 2024 Datastrato Pvt Ltd. -# This software is licensed under the Apache License version 2. -requests~=2.31.0 diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py index 61acc1ad24c..13c5d41abb4 100644 --- a/clients/client-python/setup.py +++ b/clients/client-python/setup.py @@ -2,6 +2,7 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import io import os from setuptools import find_packages, setup @@ -10,21 +11,13 @@ def read(*paths, **kwargs): content = "" with io.open( - os.path.join(os.path.dirname(__file__), *paths), - encoding=kwargs.get("encoding", "utf8"), + os.path.join(os.path.dirname(__file__), *paths), + encoding=kwargs.get("encoding", "utf8"), ) as open_file: content = open_file.read().strip() return content -def read_requirements(path): - return [ - line.strip() - for line in read(path).split("\n") - if not line.startswith(('"', "#", "-", "git+")) - ] - - setup( name="gravitino", version="0.0.1", @@ -34,7 +27,7 @@ def read_requirements(path): long_description_content_type="text/markdown", author="datastrato", packages=find_packages(include=["gravitino", ".*"]), - install_requires=read_requirements("requirements.txt"), + install_requires=[], extras_require={ "dev": [ "pytest~=8.0.1", From 33d0ae67fadb2ad7fb00b8a8915f304d99125e9b Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Wed, 27 Mar 2024 14:36:21 +0100 Subject: [PATCH 08/17] wip --- clients/client-python/gravitino/service.py | 23 +++++++++---------- .../client-python/gravitino/utils/__init__.py | 1 + .../gravitino/utils/http_client.py | 3 ++- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/clients/client-python/gravitino/service.py b/clients/client-python/gravitino/service.py index 59f8bba829d..85664f2aa3c 100644 --- a/clients/client-python/gravitino/service.py +++ b/clients/client-python/gravitino/service.py @@ -3,8 +3,7 @@ This software is licensed under the Apache License version 2. """ -from gravitino.utils.http_client import HTTPClient, unpack -from gravitino.typing import JSON_ro +from gravitino.utils import HTTPClient, unpack, Response from gravitino.constants import TIMEOUT @@ -17,37 +16,37 @@ def __init__( self.http_client = HTTPClient(url, timeout=timeout) @unpack("version") - def get_version(self) -> JSON_ro: + def get_version(self) -> Response: return self.http_client.get("/version") @unpack("metalakes") - def get_metalakes(self) -> JSON_ro: + def get_metalakes(self) -> Response: return self.http_client.get("/metalakes") @unpack("metalake") - def get_metalake(self, metalake: str) -> JSON_ro: + def get_metalake(self, metalake: str) -> Response: return self.http_client.get(f"/metalakes/{metalake}") @unpack("identifiers") - def get_catalogs(self, metalake: str) -> JSON_ro: + def get_catalogs(self, metalake: str) -> Response: return self.http_client.get(f"/metalakes/{metalake}/catalogs/") @unpack("catalog") - def get_catalog(self, metalake: str, catalog: str) -> JSON_ro: + def get_catalog(self, metalake: str, catalog: str) -> Response: return self.http_client.get(f"/metalakes/{metalake}/catalogs/{catalog}") @unpack("identifiers") - def get_schemas(self, metalake: str, catalog: str) -> JSON_ro: + def get_schemas(self, metalake: str, catalog: str) -> Response: return self.http_client.get(f"/metalakes/{metalake}/catalogs/{catalog}/schemas") @unpack("schema") - def get_schema(self, metalake: str, catalog: str, schema: str) -> JSON_ro: + def get_schema(self, metalake: str, catalog: str, schema: str) -> Response: return self.http_client.get( f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}" ) @unpack("identifiers") - def get_tables(self, metalake: str, catalog: str, schema: str) -> JSON_ro: + def get_tables(self, metalake: str, catalog: str, schema: str) -> Response: return self.http_client.get( f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables" ) @@ -55,7 +54,7 @@ def get_tables(self, metalake: str, catalog: str, schema: str) -> JSON_ro: @unpack("table") def get_table( self, metalake: str, catalog: str, schema: str, table: str - ) -> JSON_ro: + ) -> Response: return self.http_client.get( f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}" ) @@ -63,7 +62,7 @@ def get_table( @unpack("names") def get_partitions( self, metalake: str, catalog: str, schema: str, table: str - ) -> JSON_ro: + ) -> Response: return self.http_client.get( f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables/{table}/partitions" ) diff --git a/clients/client-python/gravitino/utils/__init__.py b/clients/client-python/gravitino/utils/__init__.py index 5779a3ad252..5ded219b747 100644 --- a/clients/client-python/gravitino/utils/__init__.py +++ b/clients/client-python/gravitino/utils/__init__.py @@ -2,3 +2,4 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ +from gravitino.utils.http_client import Response, HTTPClient, unpack diff --git a/clients/client-python/gravitino/utils/http_client.py b/clients/client-python/gravitino/utils/http_client.py index e914babbbf4..bb01c17fabc 100644 --- a/clients/client-python/gravitino/utils/http_client.py +++ b/clients/client-python/gravitino/utils/http_client.py @@ -8,6 +8,7 @@ from urllib.error import HTTPError import json as _json +from gravitino.typing import JSON_ro from gravitino.utils.exceptions import handle_error from gravitino.constants import TIMEOUT @@ -144,7 +145,7 @@ def put(self, endpoint, json=None, **kwargs): def unpack(path: str): def decorator(func): - def wrapper(*args, **kwargs): + def wrapper(*args, **kwargs) -> JSON_ro: resp = func(*args, **kwargs) rv = resp.json() for p in path.split("."): From e14faf1e9e216dd73e785afa0a51a7281033a1a0 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Sat, 30 Mar 2024 11:00:27 +0100 Subject: [PATCH 09/17] add unittests --- clients/client-python/Makefile | 13 ++++++++++ clients/client-python/gravitino/__init__.py | 9 ++++++- clients/client-python/requirements-dev.txt | 0 clients/client-python/requirements.txt | 0 clients/client-python/setup.py | 25 +++++-------------- clients/client-python/tests/__init__.py | 4 +++ .../tests/test_gravitino_client.py | 13 ++++++++++ 7 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 clients/client-python/Makefile create mode 100644 clients/client-python/requirements-dev.txt create mode 100644 clients/client-python/requirements.txt create mode 100644 clients/client-python/tests/__init__.py create mode 100644 clients/client-python/tests/test_gravitino_client.py diff --git a/clients/client-python/Makefile b/clients/client-python/Makefile new file mode 100644 index 00000000000..72d231ff978 --- /dev/null +++ b/clients/client-python/Makefile @@ -0,0 +1,13 @@ +.PHONY: mkvenv install test clean + +mkvenv: + python -m venv venv && source ./venv/bin/activate && pip install -U pip + +install: + source ./venv/bin/activate && pip install -e . + +test: + source ./venv/bin/activate && python -m unittest + +clean: + rm -rf venv diff --git a/clients/client-python/gravitino/__init__.py b/clients/client-python/gravitino/__init__.py index af49bcc12d9..fdd4d199d30 100644 --- a/clients/client-python/gravitino/__init__.py +++ b/clients/client-python/gravitino/__init__.py @@ -3,4 +3,11 @@ This software is licensed under the Apache License version 2. """ -from gravitino.gravitino_client import GravitinoClient, gravitino_metalake +from gravitino.gravitino_client import ( + GravitinoClient, + gravitino_metalake, + MetaLake, + Catalog, + Schema, + Table, +) diff --git a/clients/client-python/requirements-dev.txt b/clients/client-python/requirements-dev.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/clients/client-python/requirements.txt b/clients/client-python/requirements.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py index 13c5d41abb4..e130f2ed0fd 100644 --- a/clients/client-python/setup.py +++ b/clients/client-python/setup.py @@ -3,34 +3,21 @@ This software is licensed under the Apache License version 2. """ -import io -import os from setuptools import find_packages, setup -def read(*paths, **kwargs): - content = "" - with io.open( - os.path.join(os.path.dirname(__file__), *paths), - encoding=kwargs.get("encoding", "utf8"), - ) as open_file: - content = open_file.read().strip() - return content - - setup( name="gravitino", - version="0.0.1", description="project description TBD", - url="https://github.com/datastrato/gravitino", - long_description=read("README.md"), + version="0.0.1", + long_description=open("README.md").read(), long_description_content_type="text/markdown", + url="https://github.com/datastrato/gravitino", author="datastrato", + python_requires=">=3.8", packages=find_packages(include=["gravitino", ".*"]), - install_requires=[], + install_requires=open("requirements.txt").read(), extras_require={ - "dev": [ - "pytest~=8.0.1", - ] + "dev": open("requirements-dev.txt").read(), }, ) diff --git a/clients/client-python/tests/__init__.py b/clients/client-python/tests/__init__.py new file mode 100644 index 00000000000..5779a3ad252 --- /dev/null +++ b/clients/client-python/tests/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" diff --git a/clients/client-python/tests/test_gravitino_client.py b/clients/client-python/tests/test_gravitino_client.py new file mode 100644 index 00000000000..d06266f9a8c --- /dev/null +++ b/clients/client-python/tests/test_gravitino_client.py @@ -0,0 +1,13 @@ +import unittest +from unittest.mock import patch + +from gravitino import GravitinoClient, gravitino_metalake, MetaLake + + +@patch('gravitino.service._Service.get_version', return_value={'foo': 'bar'}) +class TestGravitinoClient(unittest.TestCase): + def setUp(self): + self.client = GravitinoClient("http://localhost:8090") + + def test_gravitino_version(self, *args): + self.assertEquals(self.client.version, {'foo': 'bar'}) From ee4403dfffedf89f1f349503827d74b07703e720 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Sat, 30 Mar 2024 11:21:24 +0100 Subject: [PATCH 10/17] Rm .gitignore --- clients/client-python/.gitignore | 160 ------------------------------- 1 file changed, 160 deletions(-) delete mode 100644 clients/client-python/.gitignore diff --git a/clients/client-python/.gitignore b/clients/client-python/.gitignore deleted file mode 100644 index 6769e21d99a..00000000000 --- a/clients/client-python/.gitignore +++ /dev/null @@ -1,160 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file From a9a327dcd1ceb69073b637eb0e474b0a587a9058 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Sat, 30 Mar 2024 11:41:24 +0100 Subject: [PATCH 11/17] header of license --- clients/client-python/tests/test_gravitino_client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clients/client-python/tests/test_gravitino_client.py b/clients/client-python/tests/test_gravitino_client.py index d06266f9a8c..9eeb26e5eaf 100644 --- a/clients/client-python/tests/test_gravitino_client.py +++ b/clients/client-python/tests/test_gravitino_client.py @@ -1,3 +1,7 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" import unittest from unittest.mock import patch From 655bb61cd6b8e88303021dfb9479de80dbcf4a14 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Sat, 30 Mar 2024 11:46:51 +0100 Subject: [PATCH 12/17] license --- clients/client-python/Makefile | 2 ++ clients/client-python/requirements-dev.txt | 2 ++ clients/client-python/requirements.txt | 2 ++ 3 files changed, 6 insertions(+) diff --git a/clients/client-python/Makefile b/clients/client-python/Makefile index 72d231ff978..f769e5ac76a 100644 --- a/clients/client-python/Makefile +++ b/clients/client-python/Makefile @@ -1,3 +1,5 @@ +# Copyright 2024 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. .PHONY: mkvenv install test clean mkvenv: diff --git a/clients/client-python/requirements-dev.txt b/clients/client-python/requirements-dev.txt index e69de29bb2d..2cfa42afe23 100644 --- a/clients/client-python/requirements-dev.txt +++ b/clients/client-python/requirements-dev.txt @@ -0,0 +1,2 @@ +# Copyright 2024 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. \ No newline at end of file diff --git a/clients/client-python/requirements.txt b/clients/client-python/requirements.txt index e69de29bb2d..2cfa42afe23 100644 --- a/clients/client-python/requirements.txt +++ b/clients/client-python/requirements.txt @@ -0,0 +1,2 @@ +# Copyright 2024 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. \ No newline at end of file From c39e26acee31c7f9c168f31d986e6830a0876b97 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Sat, 30 Mar 2024 14:42:50 +0100 Subject: [PATCH 13/17] Revert "Rm .gitignore" This reverts commit ee4403dfffedf89f1f349503827d74b07703e720. --- clients/client-python/.gitignore | 160 +++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 clients/client-python/.gitignore diff --git a/clients/client-python/.gitignore b/clients/client-python/.gitignore new file mode 100644 index 00000000000..6769e21d99a --- /dev/null +++ b/clients/client-python/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file From 834c6f353e64a04920e754edfa0396d061bfb115 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Sat, 30 Mar 2024 15:58:45 +0100 Subject: [PATCH 14/17] UT done --- .../client-python/gravitino/utils/__init__.py | 1 + .../gravitino/utils/http_client.py | 14 --- clients/client-python/tests/fixtures.py | 104 ++++++++++++++++++ .../tests/test_gravitino_client.py | 80 +++++++++++++- 4 files changed, 180 insertions(+), 19 deletions(-) create mode 100644 clients/client-python/tests/fixtures.py diff --git a/clients/client-python/gravitino/utils/__init__.py b/clients/client-python/gravitino/utils/__init__.py index 5ded219b747..03295d89182 100644 --- a/clients/client-python/gravitino/utils/__init__.py +++ b/clients/client-python/gravitino/utils/__init__.py @@ -2,4 +2,5 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + from gravitino.utils.http_client import Response, HTTPClient, unpack diff --git a/clients/client-python/gravitino/utils/http_client.py b/clients/client-python/gravitino/utils/http_client.py index bb01c17fabc..870d2c2bc31 100644 --- a/clients/client-python/gravitino/utils/http_client.py +++ b/clients/client-python/gravitino/utils/http_client.py @@ -87,20 +87,6 @@ def _mask_auth_headers(self, headers): def _make_request(self, opener, request, timeout=None): timeout = timeout or self.timeout - # logger.debug( - # """ - # ==========================[QUERY]=============================== - # method: {} - # url: {} - # headers: {} - # body: {} - # ==========================[QUERY]===============================""".format( # noqa - # request.get_method(), - # request.get_full_url(), - # self._mask_auth_headers(dict(request.header_items())), - # request.data, - # ) - # ) try: return opener.open(request, timeout=timeout) except HTTPError as err: diff --git a/clients/client-python/tests/fixtures.py b/clients/client-python/tests/fixtures.py new file mode 100644 index 00000000000..ba01ea33402 --- /dev/null +++ b/clients/client-python/tests/fixtures.py @@ -0,0 +1,104 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" + +services_version = { + "version": "0.4.0", + "compileDate": "06/02/2024 08:37:11", + "gitCommit": "ae87dbdef5a749cdbed66d7f0e841cc809ad2510", +} +services_get_metalakes = [ + { + "name": "metalake_demo", + "comment": "comment", + "audit": {"creator": "anonymous", "createTime": "2024-03-30T13:49:53.382Z"}, + } +] +services_get_metalake = { + "name": "metalake_demo", + "comment": "comment", + "audit": {"creator": "anonymous", "createTime": "2024-03-30T13:49:53.382Z"}, +} +services_get_catalogs = [ + {"namespace": ["metalake_demo"], "name": "catalog_hive"}, + {"namespace": ["metalake_demo"], "name": "catalog_iceberg"}, + {"namespace": ["metalake_demo"], "name": "catalog_postgres"}, +] +services_get_catalog = {"namespace": ["metalake_demo"], "name": "catalog_hive"} +services_get_schemas = [ + {"namespace": ["metalake_demo", "catalog_hive"], "name": "default"}, + {"namespace": ["metalake_demo", "catalog_hive"], "name": "sales"}, +] +services_get_schema = {"namespace": ["metalake_demo", "catalog_hive"], "name": "sales"} +services_get_tables = [ + {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "categories"}, + {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "customers"}, + {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "products"}, + {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "sales"}, + {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "stores"}, +] +services_get_table = { + "name": "sales", + "comment": "", + "columns": [ + { + "name": "sale_id", + "type": "integer", + "nullable": True, + "autoIncrement": False, + }, + { + "name": "employee_id", + "type": "integer", + "nullable": True, + "autoIncrement": False, + }, + { + "name": "store_id", + "type": "integer", + "nullable": True, + "autoIncrement": False, + }, + { + "name": "product_id", + "type": "integer", + "nullable": True, + "autoIncrement": False, + }, + { + "name": "customer_id", + "type": "integer", + "nullable": True, + "autoIncrement": False, + }, + {"name": "sold", "type": "date", "nullable": True, "autoIncrement": False}, + { + "name": "quantity", + "type": "integer", + "nullable": True, + "autoIncrement": False, + }, + { + "name": "total_amount", + "type": "decimal(10,2)", + "nullable": True, + "autoIncrement": False, + }, + ], + "properties": { + "input-format": "org.apache.hadoop.mapred.TextInputFormat", + "transient_lastDdlTime": "1711806631", + "output-format": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "location": "hdfs://hive:9000/user/hive/warehouse/sales.db/sales", + "table-type": "MANAGED_TABLE", + "serde-lib": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", + "STATS_GENERATED_VIA_STATS_TASK": "workaround for potential lack of HIVE-12730", + "serde-name": "sales", + }, + "audit": {"creator": "anonymous", "createTime": "2024-03-30T13:50:31.289Z"}, + "distribution": {"strategy": "none", "number": 0, "funcArgs": []}, + "sortOrders": [], + "partitioning": [], + "indexes": [], +} diff --git a/clients/client-python/tests/test_gravitino_client.py b/clients/client-python/tests/test_gravitino_client.py index 9eeb26e5eaf..c79c7341bd3 100644 --- a/clients/client-python/tests/test_gravitino_client.py +++ b/clients/client-python/tests/test_gravitino_client.py @@ -2,16 +2,86 @@ Copyright 2024 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2. """ + import unittest from unittest.mock import patch -from gravitino import GravitinoClient, gravitino_metalake, MetaLake +from gravitino import GravitinoClient, gravitino_metalake +from . import fixtures -@patch('gravitino.service._Service.get_version', return_value={'foo': 'bar'}) +@patch("gravitino.service._Service.get_version", return_value=fixtures.services_version) +@patch( + "gravitino.service._Service.get_metalakes", + return_value=fixtures.services_get_metalakes, +) +@patch( + "gravitino.service._Service.get_metalake", + return_value=fixtures.services_get_metalake, +) +@patch( + "gravitino.service._Service.get_catalogs", + return_value=fixtures.services_get_catalogs, +) +@patch( + "gravitino.service._Service.get_catalog", + return_value=fixtures.services_get_catalog, +) +@patch( + "gravitino.service._Service.get_schemas", + return_value=fixtures.services_get_schemas, +) +@patch( + "gravitino.service._Service.get_schema", + return_value=fixtures.services_get_schema, +) +@patch( + "gravitino.service._Service.get_tables", + return_value=fixtures.services_get_tables, +) +@patch( + "gravitino.service._Service.get_table", + return_value=fixtures.services_get_table, +) class TestGravitinoClient(unittest.TestCase): def setUp(self): - self.client = GravitinoClient("http://localhost:8090") + self.client = GravitinoClient("http://localhost:9000") + + def test_version(self, *args): + self.assertIn("version", list(self.client.version.keys())) + + def test_get_metalakes(self, *args): + metalakes = self.client.get_metalakes() + self.assertEqual(len(metalakes), 1) + self.assertEqual(metalakes[0].name, "metalake_demo") + + def test_get_metalake(self, *args): + metalake = self.client.get_metalake("metalake_demo") + self.assertEqual(metalake.name, "metalake_demo") + + def test_dynamic_properties(self, *args): + metalake = self.client.get_metalake("metalake_demo") + self.assertIn("catalog_hive", dir(metalake)) + self.assertIn("catalog_iceberg", dir(metalake)) + self.assertIn("catalog_postgres", dir(metalake)) + self.assertEqual(metalake.catalog_hive.name, "catalog_hive") + self.assertEqual(metalake.catalog_hive.sales.name, "sales") + self.assertEqual(metalake.catalog_hive.sales.sales.info().get("name"), "sales") + - def test_gravitino_version(self, *args): - self.assertEquals(self.client.version, {'foo': 'bar'}) +@patch( + "gravitino.service._Service.get_metalake", + return_value=fixtures.services_get_metalake, +) +@patch( + "gravitino.service._Service.get_catalogs", + return_value=fixtures.services_get_catalogs, +) +@patch( + "gravitino.service._Service.get_catalog", + return_value=fixtures.services_get_catalog, +) +class TestGravitinoMetalake(unittest.TestCase): + def test_gravitino_metalake(self, *args): + metalake = gravitino_metalake("http://localhost:9000", "metalake_demo") + self.assertEqual(metalake.name, "metalake_demo") From ff6850caebfcd417568f0bb416172b8586be15a4 Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Mon, 1 Apr 2024 13:33:10 +0200 Subject: [PATCH 15/17] rename methods --- .../gravitino/gravitino_client.py | 8 ++++---- clients/client-python/gravitino/service.py | 10 +++++----- clients/client-python/tests/fixtures.py | 8 ++++---- .../tests/test_gravitino_client.py | 20 +++++++++---------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/clients/client-python/gravitino/gravitino_client.py b/clients/client-python/gravitino/gravitino_client.py index c0deef555a9..b279ed8f180 100644 --- a/clients/client-python/gravitino/gravitino_client.py +++ b/clients/client-python/gravitino/gravitino_client.py @@ -15,7 +15,7 @@ def __init__(self, payload: JSON_ro): self.name = payload.get("name") self.payload = payload self.service = service["service"] - self.catalogs = self.service.get_catalogs(self.name) + self.catalogs = self.service.list_catalogs(self.name) def __repr__(self): return f"MetaLake<{self.name}>" @@ -34,7 +34,7 @@ def __init__(self, metalake_name: str, catalog_name: str): self.catalog_name = catalog_name self.name = catalog_name self.service = service["service"] - self.schemas = self.service.get_schemas(metalake_name, catalog_name) + self.schemas = self.service.list_schemas(metalake_name, catalog_name) def __repr__(self): return f"Catalog<{self.name}>" @@ -54,7 +54,7 @@ def __init__(self, metalake_name: str, catalog_name: str, schema_name: str): self.schema_name = schema_name self.name = schema_name self.service = service["service"] - self.tables = self.service.get_tables(metalake_name, catalog_name, schema_name) + self.tables = self.service.list_tables(metalake_name, catalog_name, schema_name) def __repr__(self): return f"Schema<{self.name}>" @@ -141,7 +141,7 @@ def version(self): return self.service.get_version() def get_metalakes(self) -> [MetaLake]: - return [MetaLake(metalake) for metalake in self.service.get_metalakes()] + return [MetaLake(metalake) for metalake in self.service.list_metalakes()] def get_metalake(self, metalake: str) -> MetaLake: return MetaLake(self.service.get_metalake(metalake)) diff --git a/clients/client-python/gravitino/service.py b/clients/client-python/gravitino/service.py index 85664f2aa3c..9bd4cd6dfbe 100644 --- a/clients/client-python/gravitino/service.py +++ b/clients/client-python/gravitino/service.py @@ -20,7 +20,7 @@ def get_version(self) -> Response: return self.http_client.get("/version") @unpack("metalakes") - def get_metalakes(self) -> Response: + def list_metalakes(self) -> Response: return self.http_client.get("/metalakes") @unpack("metalake") @@ -28,7 +28,7 @@ def get_metalake(self, metalake: str) -> Response: return self.http_client.get(f"/metalakes/{metalake}") @unpack("identifiers") - def get_catalogs(self, metalake: str) -> Response: + def list_catalogs(self, metalake: str) -> Response: return self.http_client.get(f"/metalakes/{metalake}/catalogs/") @unpack("catalog") @@ -36,7 +36,7 @@ def get_catalog(self, metalake: str, catalog: str) -> Response: return self.http_client.get(f"/metalakes/{metalake}/catalogs/{catalog}") @unpack("identifiers") - def get_schemas(self, metalake: str, catalog: str) -> Response: + def list_schemas(self, metalake: str, catalog: str) -> Response: return self.http_client.get(f"/metalakes/{metalake}/catalogs/{catalog}/schemas") @unpack("schema") @@ -46,7 +46,7 @@ def get_schema(self, metalake: str, catalog: str, schema: str) -> Response: ) @unpack("identifiers") - def get_tables(self, metalake: str, catalog: str, schema: str) -> Response: + def list_tables(self, metalake: str, catalog: str, schema: str) -> Response: return self.http_client.get( f"/metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/tables" ) @@ -60,7 +60,7 @@ def get_table( ) @unpack("names") - def get_partitions( + def list_partitions( self, metalake: str, catalog: str, schema: str, table: str ) -> Response: return self.http_client.get( diff --git a/clients/client-python/tests/fixtures.py b/clients/client-python/tests/fixtures.py index ba01ea33402..dbf47969934 100644 --- a/clients/client-python/tests/fixtures.py +++ b/clients/client-python/tests/fixtures.py @@ -8,7 +8,7 @@ "compileDate": "06/02/2024 08:37:11", "gitCommit": "ae87dbdef5a749cdbed66d7f0e841cc809ad2510", } -services_get_metalakes = [ +services_list_metalakes = [ { "name": "metalake_demo", "comment": "comment", @@ -20,18 +20,18 @@ "comment": "comment", "audit": {"creator": "anonymous", "createTime": "2024-03-30T13:49:53.382Z"}, } -services_get_catalogs = [ +services_list_catalogs = [ {"namespace": ["metalake_demo"], "name": "catalog_hive"}, {"namespace": ["metalake_demo"], "name": "catalog_iceberg"}, {"namespace": ["metalake_demo"], "name": "catalog_postgres"}, ] services_get_catalog = {"namespace": ["metalake_demo"], "name": "catalog_hive"} -services_get_schemas = [ +services_list_schemas = [ {"namespace": ["metalake_demo", "catalog_hive"], "name": "default"}, {"namespace": ["metalake_demo", "catalog_hive"], "name": "sales"}, ] services_get_schema = {"namespace": ["metalake_demo", "catalog_hive"], "name": "sales"} -services_get_tables = [ +services_list_tables = [ {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "categories"}, {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "customers"}, {"namespace": ["metalake_demo", "catalog_hive", "sales"], "name": "products"}, diff --git a/clients/client-python/tests/test_gravitino_client.py b/clients/client-python/tests/test_gravitino_client.py index c79c7341bd3..39b6cc2edfd 100644 --- a/clients/client-python/tests/test_gravitino_client.py +++ b/clients/client-python/tests/test_gravitino_client.py @@ -12,32 +12,32 @@ @patch("gravitino.service._Service.get_version", return_value=fixtures.services_version) @patch( - "gravitino.service._Service.get_metalakes", - return_value=fixtures.services_get_metalakes, + "gravitino.service._Service.list_metalakes", + return_value=fixtures.services_list_metalakes, ) @patch( "gravitino.service._Service.get_metalake", return_value=fixtures.services_get_metalake, ) @patch( - "gravitino.service._Service.get_catalogs", - return_value=fixtures.services_get_catalogs, + "gravitino.service._Service.list_catalogs", + return_value=fixtures.services_list_catalogs, ) @patch( "gravitino.service._Service.get_catalog", return_value=fixtures.services_get_catalog, ) @patch( - "gravitino.service._Service.get_schemas", - return_value=fixtures.services_get_schemas, + "gravitino.service._Service.list_schemas", + return_value=fixtures.services_list_schemas, ) @patch( "gravitino.service._Service.get_schema", return_value=fixtures.services_get_schema, ) @patch( - "gravitino.service._Service.get_tables", - return_value=fixtures.services_get_tables, + "gravitino.service._Service.list_tables", + return_value=fixtures.services_list_tables, ) @patch( "gravitino.service._Service.get_table", @@ -74,8 +74,8 @@ def test_dynamic_properties(self, *args): return_value=fixtures.services_get_metalake, ) @patch( - "gravitino.service._Service.get_catalogs", - return_value=fixtures.services_get_catalogs, + "gravitino.service._Service.list_catalogs", + return_value=fixtures.services_list_catalogs, ) @patch( "gravitino.service._Service.get_catalog", From c19f12abe2a739f3f279f30f3f885346de42ec8d Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Mon, 1 Apr 2024 21:47:42 +0200 Subject: [PATCH 16/17] UTs --- .../gravitino/gravitino_client.py | 9 +++ .../tests/test_gravitino_client.py | 67 ++++++------------- clients/client-python/tests/utils.py | 49 ++++++++++++++ 3 files changed, 77 insertions(+), 48 deletions(-) create mode 100644 clients/client-python/tests/utils.py diff --git a/clients/client-python/gravitino/gravitino_client.py b/clients/client-python/gravitino/gravitino_client.py index b279ed8f180..9ef433ff359 100644 --- a/clients/client-python/gravitino/gravitino_client.py +++ b/clients/client-python/gravitino/gravitino_client.py @@ -27,6 +27,9 @@ def __getattr__(self, catalog_name): def __dir__(self): return [catalog["name"] for catalog in self.catalogs] + def __contains__(self, item): + return item in dir(self) + class Catalog: def __init__(self, metalake_name: str, catalog_name: str): @@ -46,6 +49,9 @@ def __getattr__(self, schema_name): def __dir__(self): return [schema["name"] for schema in self.schemas] + def __contains__(self, item): + return item in dir(self) + class Schema: def __init__(self, metalake_name: str, catalog_name: str, schema_name: str): @@ -68,6 +74,9 @@ def __getattr__(self, table_name): def __dir__(self): return [table["name"] for table in self.tables] + def __contains__(self, item): + return item in dir(self) + class Table: def __init__( diff --git a/clients/client-python/tests/test_gravitino_client.py b/clients/client-python/tests/test_gravitino_client.py index 39b6cc2edfd..ad497b2ba7f 100644 --- a/clients/client-python/tests/test_gravitino_client.py +++ b/clients/client-python/tests/test_gravitino_client.py @@ -4,45 +4,12 @@ """ import unittest -from unittest.mock import patch from gravitino import GravitinoClient, gravitino_metalake -from . import fixtures +from .utils import services_fixtures -@patch("gravitino.service._Service.get_version", return_value=fixtures.services_version) -@patch( - "gravitino.service._Service.list_metalakes", - return_value=fixtures.services_list_metalakes, -) -@patch( - "gravitino.service._Service.get_metalake", - return_value=fixtures.services_get_metalake, -) -@patch( - "gravitino.service._Service.list_catalogs", - return_value=fixtures.services_list_catalogs, -) -@patch( - "gravitino.service._Service.get_catalog", - return_value=fixtures.services_get_catalog, -) -@patch( - "gravitino.service._Service.list_schemas", - return_value=fixtures.services_list_schemas, -) -@patch( - "gravitino.service._Service.get_schema", - return_value=fixtures.services_get_schema, -) -@patch( - "gravitino.service._Service.list_tables", - return_value=fixtures.services_list_tables, -) -@patch( - "gravitino.service._Service.get_table", - return_value=fixtures.services_get_table, -) +@services_fixtures class TestGravitinoClient(unittest.TestCase): def setUp(self): self.client = GravitinoClient("http://localhost:9000") @@ -58,6 +25,22 @@ def test_get_metalakes(self, *args): def test_get_metalake(self, *args): metalake = self.client.get_metalake("metalake_demo") self.assertEqual(metalake.name, "metalake_demo") + self.assertIn("catalog_hive", metalake) + + def test_get_catalog(self, *args): + catalog = self.client.get_metalake("metalake_demo").catalog_hive + self.assertEqual(catalog.name, "catalog_hive") + self.assertIn("sales", catalog) + + def test_get_schema(self, *args): + schema = self.client.get_metalake("metalake_demo").catalog_hive.sales + self.assertEqual(schema.name, "sales") + self.assertIn("sales", schema) + + def test_get_table(self, *args): + table = self.client.get_metalake("metalake_demo").catalog_hive.sales.sales + self.assertEqual(table.name, "sales") + self.assertEqual(table.info().get("name"), "sales") def test_dynamic_properties(self, *args): metalake = self.client.get_metalake("metalake_demo") @@ -66,21 +49,9 @@ def test_dynamic_properties(self, *args): self.assertIn("catalog_postgres", dir(metalake)) self.assertEqual(metalake.catalog_hive.name, "catalog_hive") self.assertEqual(metalake.catalog_hive.sales.name, "sales") - self.assertEqual(metalake.catalog_hive.sales.sales.info().get("name"), "sales") -@patch( - "gravitino.service._Service.get_metalake", - return_value=fixtures.services_get_metalake, -) -@patch( - "gravitino.service._Service.list_catalogs", - return_value=fixtures.services_list_catalogs, -) -@patch( - "gravitino.service._Service.get_catalog", - return_value=fixtures.services_get_catalog, -) +@services_fixtures class TestGravitinoMetalake(unittest.TestCase): def test_gravitino_metalake(self, *args): metalake = gravitino_metalake("http://localhost:9000", "metalake_demo") diff --git a/clients/client-python/tests/utils.py b/clients/client-python/tests/utils.py new file mode 100644 index 00000000000..9953e929b71 --- /dev/null +++ b/clients/client-python/tests/utils.py @@ -0,0 +1,49 @@ +""" +Copyright 2024 Datastrato Pvt Ltd. +This software is licensed under the Apache License version 2. +""" + +from unittest.mock import patch +from . import fixtures + + +def services_fixtures(cls): + @patch( + "gravitino.service._Service.get_version", return_value=fixtures.services_version + ) + @patch( + "gravitino.service._Service.list_metalakes", + return_value=fixtures.services_list_metalakes, + ) + @patch( + "gravitino.service._Service.get_metalake", + return_value=fixtures.services_get_metalake, + ) + @patch( + "gravitino.service._Service.list_catalogs", + return_value=fixtures.services_list_catalogs, + ) + @patch( + "gravitino.service._Service.get_catalog", + return_value=fixtures.services_get_catalog, + ) + @patch( + "gravitino.service._Service.list_schemas", + return_value=fixtures.services_list_schemas, + ) + @patch( + "gravitino.service._Service.get_schema", + return_value=fixtures.services_get_schema, + ) + @patch( + "gravitino.service._Service.list_tables", + return_value=fixtures.services_list_tables, + ) + @patch( + "gravitino.service._Service.get_table", + return_value=fixtures.services_get_table, + ) + class Wrapper(cls): + pass + + return Wrapper From 036655e1d24ae00e36cd23d0f79fdf8566c46e0e Mon Sep 17 00:00:00 2001 From: Yongjie Zhao Date: Mon, 1 Apr 2024 22:06:15 +0200 Subject: [PATCH 17/17] remove protocal and port --- .../gravitino/gravitino_client.py | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/clients/client-python/gravitino/gravitino_client.py b/clients/client-python/gravitino/gravitino_client.py index 9ef433ff359..a8f97bb85c7 100644 --- a/clients/client-python/gravitino/gravitino_client.py +++ b/clients/client-python/gravitino/gravitino_client.py @@ -3,18 +3,15 @@ This software is licensed under the Apache License version 2. """ -import re - from gravitino.constants import TIMEOUT from gravitino.service import initialize_service, service -from gravitino.typing import JSON_ro class MetaLake: - def __init__(self, payload: JSON_ro): - self.name = payload.get("name") - self.payload = payload + def __init__(self, metalake_name: str): + self.name = metalake_name self.service = service["service"] + self.metalake = self.service.get_metalake(self.name) self.catalogs = self.service.list_catalogs(self.name) def __repr__(self): @@ -103,21 +100,11 @@ def __init__( self, host: str, *, - protocol: str = "http", - port: int = 8090, prefix: str = "/api", timeout: int = TIMEOUT, debug: bool = False, ) -> None: - if re.search(r"^https?:\/\/", host): - _host = host.rstrip("/") - else: - _host = f"{protocol}://{host.rstrip('/')}" - - if not re.search(r"[0-9]{2,5}$", _host): - _host = f"{_host}:{port}" - - _base_url = f"{_host}/{prefix.strip('/')}" + _base_url = f"{host.rstrip('/')}/{prefix.strip('/')}" initialize_service(_base_url, timeout) self.service = service["service"] self.debug = debug @@ -128,8 +115,6 @@ def initialize_metalake( host: str, metalake_name: str, *, - protocol: str = "http", - port: int = 8090, prefix: str = "/api", timeout: int = TIMEOUT, debug: bool = False, @@ -137,8 +122,6 @@ def initialize_metalake( # keep in mind, all constructors should include same interface as __init__ function client = cls( host, - protocol=protocol, - port=port, prefix=prefix, timeout=timeout, debug=debug, @@ -150,18 +133,18 @@ def version(self): return self.service.get_version() def get_metalakes(self) -> [MetaLake]: - return [MetaLake(metalake) for metalake in self.service.list_metalakes()] + return [ + MetaLake(metalake.get("name")) for metalake in self.service.list_metalakes() + ] def get_metalake(self, metalake: str) -> MetaLake: - return MetaLake(self.service.get_metalake(metalake)) + return MetaLake(metalake) def gravitino_metalake( host: str, metalake_name: str, *, - protocol: str = "http", - port: int = 8090, prefix: str = "/api", timeout: int = TIMEOUT, debug: bool = False, @@ -169,8 +152,6 @@ def gravitino_metalake( return GravitinoClient.initialize_metalake( host, metalake_name, - protocol=protocol, - port=port, prefix=prefix, timeout=timeout, debug=debug,