From 3a7233944ce7bffa5b7ae190ef573b2b0eaa9be5 Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 15:36:53 +0000 Subject: [PATCH 01/20] Text to SQL support --- fern/docs.yml | 2 + fern/docs/pages/manual/nlsql.mdx | 28 +++++++++ private_gpt/components/nlsql/__init__.py | 0 .../components/nlsql/nlsql_component.py | 62 +++++++++++++++++++ private_gpt/server/chat/chat_service.py | 20 ++++++ private_gpt/settings/settings.py | 30 +++++++++ private_gpt/ui/ui.py | 13 +++- pyproject.toml | 2 + settings.yaml | 9 +++ 9 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 fern/docs/pages/manual/nlsql.mdx create mode 100644 private_gpt/components/nlsql/__init__.py create mode 100644 private_gpt/components/nlsql/nlsql_component.py diff --git a/fern/docs.yml b/fern/docs.yml index 67021673b..c77a474e1 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -58,6 +58,8 @@ navigation: contents: - page: Vector Stores path: ./docs/pages/manual/vectordb.mdx + - page: SQL Databases + path: ./docs/pages/manual/nlsql.mdx - section: Advanced Setup contents: - page: LLM Backends diff --git a/fern/docs/pages/manual/nlsql.mdx b/fern/docs/pages/manual/nlsql.mdx new file mode 100644 index 000000000..46566f518 --- /dev/null +++ b/fern/docs/pages/manual/nlsql.mdx @@ -0,0 +1,28 @@ +## MySQL Configuration +Text-to-SQL querying has been tested for [MySQL](https://www.mysql.com/) in PrivateGPT. + +To use the text to SQL feature, set the following environment variables to allow connecting to your MySQL Server: + +```yaml +sqldatabase: + dialect: mysql + driver: pymysql + host: + user: + password: + database: + tables: +``` + +You will also need to install `pymysql`. + +`poetry add pymysql` + +## Other Databases +To get started with exploring the use of other databases, set the `sqldatabase.dialect` and `sqldatabase.driver` properties in the `settings.yaml` file. + +```yaml +sqldatabase: + dialect: + driver: pymysql +``` \ No newline at end of file diff --git a/private_gpt/components/nlsql/__init__.py b/private_gpt/components/nlsql/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/private_gpt/components/nlsql/nlsql_component.py b/private_gpt/components/nlsql/nlsql_component.py new file mode 100644 index 000000000..29d39c200 --- /dev/null +++ b/private_gpt/components/nlsql/nlsql_component.py @@ -0,0 +1,62 @@ +import logging + +from injector import inject, singleton +from llama_index import SQLDatabase, VectorStoreIndex, ServiceContext +from llama_index.indices.struct_store import SQLTableRetrieverQueryEngine +from llama_index.objects import SQLTableNodeMapping, ObjectIndex, SQLTableSchema +from sqlalchemy import ( + MetaData, +) +from sqlalchemy.engine import create_engine +from sqlalchemy.engine.base import Engine +from private_gpt.settings.settings import Settings +from urllib.parse import quote_plus + + + +logger = logging.getLogger(__name__) + +@singleton +class NLSQLComponent: + sqlalchemy_engine: Engine + sql_database: SQLDatabase + metadata_obj: MetaData + + @inject + def __init__(self, settings: Settings) -> None: + dialect=settings.sqldatabase.dialect + driver=settings.sqldatabase.driver + host=settings.sqldatabase.host + user=settings.sqldatabase.user + password=settings.sqldatabase.password + database=settings.sqldatabase.database + tables=settings.sqldatabase.tables + + engine = create_engine(f'{dialect}+{driver}://{user}:%s@{host}/{database}' % quote_plus(password)) + metadata_obj = MetaData() + metadata_obj.reflect(engine) + sql_database = SQLDatabase(engine, include_tables=tables) + + self.sqlalchemy_engine = engine + self.sql_database = sql_database + self.metadata_obj = metadata_obj + + def get_nlsql_query_engine( + self, + service_context: ServiceContext, + )-> SQLTableRetrieverQueryEngine: + table_node_mapping = SQLTableNodeMapping(self.sql_database) + table_schema_objs = [] + for table_name in self.metadata_obj.tables.keys(): + table_schema_objs.append(SQLTableSchema(table_name=table_name)) + obj_index = ObjectIndex.from_objects( + table_schema_objs, + table_node_mapping, + VectorStoreIndex, + service_context=service_context, + ) + return SQLTableRetrieverQueryEngine( + service_context=service_context, + sql_database=self.sql_database, + table_retriever=obj_index.as_retriever(similarity_top_k=1), + ) diff --git a/private_gpt/server/chat/chat_service.py b/private_gpt/server/chat/chat_service.py index ffdb3f90f..8c5516030 100644 --- a/private_gpt/server/chat/chat_service.py +++ b/private_gpt/server/chat/chat_service.py @@ -3,6 +3,7 @@ from injector import inject, singleton from llama_index import ServiceContext, StorageContext, VectorStoreIndex from llama_index.chat_engine import ContextChatEngine, SimpleChatEngine +from llama_index.indices.struct_store import SQLTableRetrieverQueryEngine from llama_index.chat_engine.types import ( BaseChatEngine, ) @@ -17,6 +18,7 @@ from private_gpt.components.vector_store.vector_store_component import ( VectorStoreComponent, ) +from private_gpt.components.nlsql.nlsql_component import NLSQLComponent from private_gpt.open_ai.extensions.context_filter import ContextFilter from private_gpt.server.chunks.chunks_service import Chunk @@ -74,9 +76,11 @@ def __init__( vector_store_component: VectorStoreComponent, embedding_component: EmbeddingComponent, node_store_component: NodeStoreComponent, + nlsql_component: NLSQLComponent, ) -> None: self.llm_service = llm_component self.vector_store_component = vector_store_component + self.nlsql_component = nlsql_component self.storage_context = StorageContext.from_defaults( vector_store=vector_store_component.vector_store, docstore=node_store_component.doc_store, @@ -116,6 +120,13 @@ def _chat_engine( service_context=self.service_context, ) + def _nlsql_engine( + self, + ) -> SQLTableRetrieverQueryEngine: + return self.nlsql_component.get_nlsql_query_engine( + service_context=self.service_context + ) + def stream_chat( self, messages: list[ChatMessage], @@ -185,3 +196,12 @@ def chat( sources = [Chunk.from_node(node) for node in wrapped_response.source_nodes] completion = Completion(response=wrapped_response.response, sources=sources) return completion + + def stream_chat_nlsql( + self, + messages: list[ChatMessage], + ) -> str: + last_message = messages[-1].content + nlsql_engine = self._nlsql_engine() + response = nlsql_engine.query(last_message) + return response diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index 8b03f6111..2ccd33bd3 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -216,6 +216,35 @@ class QdrantSettings(BaseModel): ), ) +class SQLDatabaseSettings(BaseModel): + dialect: Literal["mysql"] + driver: Literal["pymysql"] + host: str | None = Field( + None, + description="Host name of Qdrant service. If host is None, set to 'localhost'.", + ) + user: str | None = Field( + "root", + description=( + "Username to be used for accessing the SQL Database Server. If user is None, set to 'root'.", + ) + ) + password: str | None = Field( + "", + description=( + "Password to be used for accessing the SQL Database Server. If password is None, set to empty string.", + ) + ) + database: str | None = Field( + description=( + "The database name in which tables are to be queried", + ) + ) + tables: list | None = Field( + description=( + "List of tables to query into", + ) + ) class Settings(BaseModel): server: ServerSettings @@ -228,6 +257,7 @@ class Settings(BaseModel): openai: OpenAISettings vectorstore: VectorstoreSettings qdrant: QdrantSettings | None = None + sqldatabase: SQLDatabaseSettings """ diff --git a/private_gpt/ui/ui.py b/private_gpt/ui/ui.py index ad6052b1b..96168e1fc 100644 --- a/private_gpt/ui/ui.py +++ b/private_gpt/ui/ui.py @@ -30,7 +30,7 @@ SOURCES_SEPARATOR = "\n\n Sources: \n" -MODES = ["Query Docs", "Search in Docs", "LLM Chat"] +MODES = ["Query Docs", "Query Db", "Search in Docs", "LLM Chat"] class Source(BaseModel): @@ -78,7 +78,7 @@ def __init__( self._system_prompt = self._get_default_system_prompt(self.mode) def _chat(self, message: str, history: list[list[str]], mode: str, *_: Any) -> Any: - def yield_deltas(completion_gen: CompletionGen) -> Iterable[str]: + def yield_deltas(completion_gen: CompletionGen, sources: bool=True) -> Iterable[str]: full_response: str = "" stream = completion_gen.response for delta in stream: @@ -88,7 +88,7 @@ def yield_deltas(completion_gen: CompletionGen) -> Iterable[str]: full_response += delta.delta or "" yield full_response - if completion_gen.sources: + if sources and completion_gen.sources: full_response += SOURCES_SEPARATOR cur_sources = Source.curate_sources(completion_gen.sources) sources_text = "\n\n\n".join( @@ -136,6 +136,13 @@ def build_history() -> list[ChatMessage]: use_context=True, ) yield from yield_deltas(query_stream) + + case "Query Db": + query_stream = self._chat_service.stream_chat_nlsql( + messages=all_messages, + ) + yield from yield_deltas(query_stream, False) + case "LLM Chat": llm_stream = self._chat_service.stream_chat( messages=all_messages, diff --git a/pyproject.toml b/pyproject.toml index 7bb3ec5ec..6864938a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,8 @@ llama-index = { extras = ["local_models"], version = "0.9.3" } watchdog = "^3.0.0" qdrant-client = "^1.6.9" chromadb = {version = "^0.4.13", optional = true} +gradio = "4.4.1" +pymysql = "^1.1.0" [tool.poetry.group.dev.dependencies] black = "^22" diff --git a/settings.yaml b/settings.yaml index af51a7f7e..e91dfce63 100644 --- a/settings.yaml +++ b/settings.yaml @@ -44,6 +44,15 @@ vectorstore: qdrant: path: local_data/private_gpt/qdrant +sqldatabase: + dialect: mysql + driver: pymysql + host: ${HOSTNAME:localhost} + user: ${USERNAME:root} + password: ${PASSWORD:} + database: ${DATABASE:} + tables: ${TABLES:} + local: prompt_style: "llama2" llm_hf_repo_id: TheBloke/Mistral-7B-Instruct-v0.1-GGUF From c29da32f5c2beeccdc1c9e1e80459dca93cd4720 Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 16:06:55 +0000 Subject: [PATCH 02/20] poetry lock issue --- poetry.lock | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/poetry.lock b/poetry.lock index 78a4bbdeb..89cfd4565 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "accelerate" @@ -3255,22 +3255,25 @@ virtualenv = ">=20.10.0" [[package]] name = "protobuf" -version = "4.25.1" +version = "4.21.12" description = "" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "protobuf-4.25.1-cp310-abi3-win32.whl", hash = "sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7"}, - {file = "protobuf-4.25.1-cp310-abi3-win_amd64.whl", hash = "sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b"}, - {file = "protobuf-4.25.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd"}, - {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb"}, - {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7"}, - {file = "protobuf-4.25.1-cp38-cp38-win32.whl", hash = "sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd"}, - {file = "protobuf-4.25.1-cp38-cp38-win_amd64.whl", hash = "sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0"}, - {file = "protobuf-4.25.1-cp39-cp39-win32.whl", hash = "sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510"}, - {file = "protobuf-4.25.1-cp39-cp39-win_amd64.whl", hash = "sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10"}, - {file = "protobuf-4.25.1-py3-none-any.whl", hash = "sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6"}, - {file = "protobuf-4.25.1.tar.gz", hash = "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2"}, + {file = "protobuf-4.21.12-cp310-abi3-win32.whl", hash = "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1"}, + {file = "protobuf-4.21.12-cp310-abi3-win_amd64.whl", hash = "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2"}, + {file = "protobuf-4.21.12-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791"}, + {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97"}, + {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7"}, + {file = "protobuf-4.21.12-cp37-cp37m-win32.whl", hash = "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717"}, + {file = "protobuf-4.21.12-cp37-cp37m-win_amd64.whl", hash = "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574"}, + {file = "protobuf-4.21.12-cp38-cp38-win32.whl", hash = "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec"}, + {file = "protobuf-4.21.12-cp38-cp38-win_amd64.whl", hash = "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30"}, + {file = "protobuf-4.21.12-cp39-cp39-win32.whl", hash = "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc"}, + {file = "protobuf-4.21.12-cp39-cp39-win_amd64.whl", hash = "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b"}, + {file = "protobuf-4.21.12-py2.py3-none-any.whl", hash = "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5"}, + {file = "protobuf-4.21.12-py3-none-any.whl", hash = "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462"}, + {file = "protobuf-4.21.12.tar.gz", hash = "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab"}, ] [[package]] @@ -3626,6 +3629,21 @@ files = [ plugins = ["importlib-metadata"] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pymysql" +version = "1.1.0" +description = "Pure Python MySQL Driver" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyMySQL-1.1.0-py3-none-any.whl", hash = "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7"}, + {file = "PyMySQL-1.1.0.tar.gz", hash = "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96"}, +] + +[package.extras] +ed25519 = ["PyNaCl (>=1.4.0)"] +rsa = ["cryptography"] + [[package]] name = "pyparsing" version = "3.1.1" @@ -5909,4 +5927,4 @@ chroma = ["chromadb"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.12" -content-hash = "7b9e7bc6e4a9ecbab92e58423c1d6746aa731b681236b1966f986feac3101959" +content-hash = "39f04c88ec55aec91720bef5a7601d5d49bcc67afaa6414554c1dabb35d63d98" From 0ead57c7a1d459f1d26fcfc351dc863f99047763 Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 16:14:20 +0000 Subject: [PATCH 03/20] lint --- .../components/nlsql/nlsql_component.py | 46 ++++++++++--------- private_gpt/server/chat/chat_service.py | 4 +- private_gpt/settings/settings.py | 16 +++---- private_gpt/ui/ui.py | 4 +- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/private_gpt/components/nlsql/nlsql_component.py b/private_gpt/components/nlsql/nlsql_component.py index 29d39c200..b278cc841 100644 --- a/private_gpt/components/nlsql/nlsql_component.py +++ b/private_gpt/components/nlsql/nlsql_component.py @@ -1,21 +1,21 @@ import logging +from urllib.parse import quote_plus from injector import inject, singleton -from llama_index import SQLDatabase, VectorStoreIndex, ServiceContext +from llama_index import ServiceContext, SQLDatabase, VectorStoreIndex from llama_index.indices.struct_store import SQLTableRetrieverQueryEngine -from llama_index.objects import SQLTableNodeMapping, ObjectIndex, SQLTableSchema +from llama_index.objects import ObjectIndex, SQLTableNodeMapping, SQLTableSchema from sqlalchemy import ( MetaData, ) from sqlalchemy.engine import create_engine from sqlalchemy.engine.base import Engine -from private_gpt.settings.settings import Settings -from urllib.parse import quote_plus - +from private_gpt.settings.settings import Settings logger = logging.getLogger(__name__) + @singleton class NLSQLComponent: sqlalchemy_engine: Engine @@ -24,27 +24,29 @@ class NLSQLComponent: @inject def __init__(self, settings: Settings) -> None: - dialect=settings.sqldatabase.dialect - driver=settings.sqldatabase.driver - host=settings.sqldatabase.host - user=settings.sqldatabase.user - password=settings.sqldatabase.password - database=settings.sqldatabase.database - tables=settings.sqldatabase.tables - - engine = create_engine(f'{dialect}+{driver}://{user}:%s@{host}/{database}' % quote_plus(password)) - metadata_obj = MetaData() - metadata_obj.reflect(engine) - sql_database = SQLDatabase(engine, include_tables=tables) - - self.sqlalchemy_engine = engine - self.sql_database = sql_database - self.metadata_obj = metadata_obj + dialect = settings.sqldatabase.dialect + driver = settings.sqldatabase.driver + host = settings.sqldatabase.host + user = settings.sqldatabase.user + password = settings.sqldatabase.password + database = settings.sqldatabase.database + tables = settings.sqldatabase.tables + + engine = create_engine( + f"{dialect}+{driver}://{user}:%s@{host}/{database}" % quote_plus(password) + ) + metadata_obj = MetaData() + metadata_obj.reflect(engine) + sql_database = SQLDatabase(engine, include_tables=tables) + + self.sqlalchemy_engine = engine + self.sql_database = sql_database + self.metadata_obj = metadata_obj def get_nlsql_query_engine( self, service_context: ServiceContext, - )-> SQLTableRetrieverQueryEngine: + ) -> SQLTableRetrieverQueryEngine: table_node_mapping = SQLTableNodeMapping(self.sql_database) table_schema_objs = [] for table_name in self.metadata_obj.tables.keys(): diff --git a/private_gpt/server/chat/chat_service.py b/private_gpt/server/chat/chat_service.py index 8c5516030..82b51d861 100644 --- a/private_gpt/server/chat/chat_service.py +++ b/private_gpt/server/chat/chat_service.py @@ -3,22 +3,22 @@ from injector import inject, singleton from llama_index import ServiceContext, StorageContext, VectorStoreIndex from llama_index.chat_engine import ContextChatEngine, SimpleChatEngine -from llama_index.indices.struct_store import SQLTableRetrieverQueryEngine from llama_index.chat_engine.types import ( BaseChatEngine, ) from llama_index.indices.postprocessor import MetadataReplacementPostProcessor +from llama_index.indices.struct_store import SQLTableRetrieverQueryEngine from llama_index.llms import ChatMessage, MessageRole from llama_index.types import TokenGen from pydantic import BaseModel from private_gpt.components.embedding.embedding_component import EmbeddingComponent from private_gpt.components.llm.llm_component import LLMComponent +from private_gpt.components.nlsql.nlsql_component import NLSQLComponent from private_gpt.components.node_store.node_store_component import NodeStoreComponent from private_gpt.components.vector_store.vector_store_component import ( VectorStoreComponent, ) -from private_gpt.components.nlsql.nlsql_component import NLSQLComponent from private_gpt.open_ai.extensions.context_filter import ContextFilter from private_gpt.server.chunks.chunks_service import Chunk diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index 2ccd33bd3..29cb3382c 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -216,6 +216,7 @@ class QdrantSettings(BaseModel): ), ) + class SQLDatabaseSettings(BaseModel): dialect: Literal["mysql"] driver: Literal["pymysql"] @@ -227,24 +228,19 @@ class SQLDatabaseSettings(BaseModel): "root", description=( "Username to be used for accessing the SQL Database Server. If user is None, set to 'root'.", - ) + ), ) password: str | None = Field( "", description=( "Password to be used for accessing the SQL Database Server. If password is None, set to empty string.", - ) + ), ) database: str | None = Field( - description=( - "The database name in which tables are to be queried", - ) - ) - tables: list | None = Field( - description=( - "List of tables to query into", - ) + description=("The database name in which tables are to be queried",) ) + tables: list | None = Field(description=("List of tables to query into",)) + class Settings(BaseModel): server: ServerSettings diff --git a/private_gpt/ui/ui.py b/private_gpt/ui/ui.py index 96168e1fc..ae0c8ebc8 100644 --- a/private_gpt/ui/ui.py +++ b/private_gpt/ui/ui.py @@ -78,7 +78,9 @@ def __init__( self._system_prompt = self._get_default_system_prompt(self.mode) def _chat(self, message: str, history: list[list[str]], mode: str, *_: Any) -> Any: - def yield_deltas(completion_gen: CompletionGen, sources: bool=True) -> Iterable[str]: + def yield_deltas( + completion_gen: CompletionGen, sources: bool = True + ) -> Iterable[str]: full_response: str = "" stream = completion_gen.response for delta in stream: From 7edd4d9f5e14b5788e1eebe8cfed600d1f32a38c Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 17:03:51 +0000 Subject: [PATCH 04/20] Update doc with engine configuration link --- fern/docs/pages/manual/nlsql.mdx | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/fern/docs/pages/manual/nlsql.mdx b/fern/docs/pages/manual/nlsql.mdx index 46566f518..e24c16be1 100644 --- a/fern/docs/pages/manual/nlsql.mdx +++ b/fern/docs/pages/manual/nlsql.mdx @@ -7,11 +7,11 @@ To use the text to SQL feature, set the following environment variables to allow sqldatabase: dialect: mysql driver: pymysql - host: - user: - password: - database: - tables: + host: + user: + password: + database: + tables: ``` You will also need to install `pymysql`. @@ -23,6 +23,8 @@ To get started with exploring the use of other databases, set the `sqldatabase.d ```yaml sqldatabase: - dialect: - driver: pymysql -``` \ No newline at end of file + dialect: + driver: +``` + +Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for constructing the DB connection string. \ No newline at end of file From acd58401f62314a90d1f9f31eec801ff7c652eed Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 17:16:02 +0000 Subject: [PATCH 05/20] fix lint error --- private_gpt/components/nlsql/nlsql_component.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private_gpt/components/nlsql/nlsql_component.py b/private_gpt/components/nlsql/nlsql_component.py index b278cc841..bf89af358 100644 --- a/private_gpt/components/nlsql/nlsql_component.py +++ b/private_gpt/components/nlsql/nlsql_component.py @@ -49,7 +49,7 @@ def get_nlsql_query_engine( ) -> SQLTableRetrieverQueryEngine: table_node_mapping = SQLTableNodeMapping(self.sql_database) table_schema_objs = [] - for table_name in self.metadata_obj.tables.keys(): + for table_name in self.metadata_obj.tables: table_schema_objs.append(SQLTableSchema(table_name=table_name)) obj_index = ObjectIndex.from_objects( table_schema_objs, From fdb30c2576457f9e1a4d39b451cb55def94daf63 Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 18:48:13 +0000 Subject: [PATCH 06/20] Fix pydantic syntax issues --- private_gpt/settings/settings.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index 29cb3382c..372886418 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -221,25 +221,21 @@ class SQLDatabaseSettings(BaseModel): dialect: Literal["mysql"] driver: Literal["pymysql"] host: str | None = Field( - None, + "localhost", description="Host name of Qdrant service. If host is None, set to 'localhost'.", ) user: str | None = Field( "root", - description=( - "Username to be used for accessing the SQL Database Server. If user is None, set to 'root'.", - ), + description="Username to be used for accessing the SQL Database Server. If user is None, set to 'root'.", ) password: str | None = Field( "", - description=( - "Password to be used for accessing the SQL Database Server. If password is None, set to empty string.", - ), + description="Password to be used for accessing the SQL Database Server. If password is None, set to empty string.", ) database: str | None = Field( - description=("The database name in which tables are to be queried",) + description="The database name in which tables are to be queried", ) - tables: list | None = Field(description=("List of tables to query into",)) + tables: list[str] | None = Field(description="List of tables to query into") class Settings(BaseModel): From ab136ea085459b6219d8b72abebab1444aab1389 Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 19:28:37 +0000 Subject: [PATCH 07/20] Improve typing, fix mypy errors --- private_gpt/server/chat/chat_service.py | 10 +++++++--- private_gpt/settings/settings.py | 2 +- private_gpt/ui/ui.py | 8 ++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/private_gpt/server/chat/chat_service.py b/private_gpt/server/chat/chat_service.py index 82b51d861..11352a7cf 100644 --- a/private_gpt/server/chat/chat_service.py +++ b/private_gpt/server/chat/chat_service.py @@ -32,6 +32,9 @@ class CompletionGen(BaseModel): response: TokenGen sources: list[Chunk] | None = None +class SqlQueryResponse(BaseModel): + response: str + sources: None = None @dataclass class ChatEngineInput: @@ -200,8 +203,9 @@ def chat( def stream_chat_nlsql( self, messages: list[ChatMessage], - ) -> str: - last_message = messages[-1].content + ) -> SqlQueryResponse: + last_message = str(messages[-1].content) nlsql_engine = self._nlsql_engine() response = nlsql_engine.query(last_message) - return response + query = SqlQueryResponse(response=str(response)) + return query diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index 372886418..c78b8eea0 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -228,7 +228,7 @@ class SQLDatabaseSettings(BaseModel): "root", description="Username to be used for accessing the SQL Database Server. If user is None, set to 'root'.", ) - password: str | None = Field( + password: str = Field( "", description="Password to be used for accessing the SQL Database Server. If password is None, set to empty string.", ) diff --git a/private_gpt/ui/ui.py b/private_gpt/ui/ui.py index ae0c8ebc8..0fbd5e7e3 100644 --- a/private_gpt/ui/ui.py +++ b/private_gpt/ui/ui.py @@ -14,7 +14,7 @@ from private_gpt.constants import PROJECT_ROOT_PATH from private_gpt.di import global_injector -from private_gpt.server.chat.chat_service import ChatService, CompletionGen +from private_gpt.server.chat.chat_service import ChatService, CompletionGen, SqlQueryResponse from private_gpt.server.chunks.chunks_service import Chunk, ChunksService from private_gpt.server.ingest.ingest_service import IngestService from private_gpt.settings.settings import settings @@ -79,7 +79,7 @@ def __init__( def _chat(self, message: str, history: list[list[str]], mode: str, *_: Any) -> Any: def yield_deltas( - completion_gen: CompletionGen, sources: bool = True + completion_gen: CompletionGen|SqlQueryResponse, sources: bool = True ) -> Iterable[str]: full_response: str = "" stream = completion_gen.response @@ -140,10 +140,10 @@ def build_history() -> list[ChatMessage]: yield from yield_deltas(query_stream) case "Query Db": - query_stream = self._chat_service.stream_chat_nlsql( + sql_stream = self._chat_service.stream_chat_nlsql( messages=all_messages, ) - yield from yield_deltas(query_stream, False) + yield from yield_deltas(sql_stream, False) case "LLM Chat": llm_stream = self._chat_service.stream_chat( From bb78c6db565f6291dd039d65bda1cf28d220255a Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 19:33:01 +0000 Subject: [PATCH 08/20] linter loves me --- private_gpt/server/chat/chat_service.py | 2 ++ private_gpt/ui/ui.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/private_gpt/server/chat/chat_service.py b/private_gpt/server/chat/chat_service.py index 11352a7cf..fbc413cbf 100644 --- a/private_gpt/server/chat/chat_service.py +++ b/private_gpt/server/chat/chat_service.py @@ -32,10 +32,12 @@ class CompletionGen(BaseModel): response: TokenGen sources: list[Chunk] | None = None + class SqlQueryResponse(BaseModel): response: str sources: None = None + @dataclass class ChatEngineInput: system_message: ChatMessage | None = None diff --git a/private_gpt/ui/ui.py b/private_gpt/ui/ui.py index 0fbd5e7e3..1efef0b35 100644 --- a/private_gpt/ui/ui.py +++ b/private_gpt/ui/ui.py @@ -14,7 +14,11 @@ from private_gpt.constants import PROJECT_ROOT_PATH from private_gpt.di import global_injector -from private_gpt.server.chat.chat_service import ChatService, CompletionGen, SqlQueryResponse +from private_gpt.server.chat.chat_service import ( + ChatService, + CompletionGen, + SqlQueryResponse, +) from private_gpt.server.chunks.chunks_service import Chunk, ChunksService from private_gpt.server.ingest.ingest_service import IngestService from private_gpt.settings.settings import settings @@ -79,7 +83,7 @@ def __init__( def _chat(self, message: str, history: list[list[str]], mode: str, *_: Any) -> Any: def yield_deltas( - completion_gen: CompletionGen|SqlQueryResponse, sources: bool = True + completion_gen: CompletionGen | SqlQueryResponse, sources: bool = True ) -> Iterable[str]: full_response: str = "" stream = completion_gen.response From 99a973a36626380b2e6055dde0b62e8a4357e4bb Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 19:46:59 +0000 Subject: [PATCH 09/20] sqldatabase.tables in settings.yaml should be a list --- settings.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.yaml b/settings.yaml index e91dfce63..832549fe7 100644 --- a/settings.yaml +++ b/settings.yaml @@ -51,7 +51,7 @@ sqldatabase: user: ${USERNAME:root} password: ${PASSWORD:} database: ${DATABASE:} - tables: ${TABLES:} + tables: ["${TABLES:}"] local: prompt_style: "llama2" From 93211ca48681cf06add1bdd63d1d8aa6accb5203 Mon Sep 17 00:00:00 2001 From: ishaandatta Date: Wed, 13 Dec 2023 20:59:00 +0000 Subject: [PATCH 10/20] Make SQL mode optional, fix tests, improve error handling --- poetry.lock | 122 +++++++++++++++++- .../components/nlsql/nlsql_component.py | 40 +++--- private_gpt/settings/settings.py | 14 +- private_gpt/ui/ui.py | 4 +- pyproject.toml | 1 + settings.yaml | 9 +- 6 files changed, 162 insertions(+), 28 deletions(-) diff --git a/poetry.lock b/poetry.lock index 89cfd4565..ddf5924f9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -391,6 +391,70 @@ files = [ {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "cfgv" version = "3.4.0" @@ -744,6 +808,51 @@ files = [ [package.extras] toml = ["tomli"] +[[package]] +name = "cryptography" +version = "41.0.7" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, + {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, + {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, + {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +nox = ["nox"] +pep8test = ["black", "check-sdist", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + [[package]] name = "cycler" version = "0.12.1" @@ -3435,6 +3544,17 @@ files = [ [package.dependencies] pyasn1 = ">=0.4.6,<0.6.0" +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + [[package]] name = "pydantic" version = "2.5.1" @@ -5927,4 +6047,4 @@ chroma = ["chromadb"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.12" -content-hash = "39f04c88ec55aec91720bef5a7601d5d49bcc67afaa6414554c1dabb35d63d98" +content-hash = "dcf5b8018707ff1b12921112ad8250c501bacaada545dac30c00c58534c395f4" diff --git a/private_gpt/components/nlsql/nlsql_component.py b/private_gpt/components/nlsql/nlsql_component.py index bf89af358..b28854d45 100644 --- a/private_gpt/components/nlsql/nlsql_component.py +++ b/private_gpt/components/nlsql/nlsql_component.py @@ -24,24 +24,30 @@ class NLSQLComponent: @inject def __init__(self, settings: Settings) -> None: - dialect = settings.sqldatabase.dialect - driver = settings.sqldatabase.driver - host = settings.sqldatabase.host - user = settings.sqldatabase.user - password = settings.sqldatabase.password - database = settings.sqldatabase.database - tables = settings.sqldatabase.tables + if settings.sqlmode.enabled: + dialect = settings.sqlmode.dialect + driver = settings.sqlmode.driver + host = settings.sqlmode.db_host + user = settings.sqlmode.db_user + password = settings.sqlmode.db_password + database = settings.sqlmode.database + tables = settings.sqlmode.tables + try: + engine = create_engine( + f"{dialect}+{driver}://{user}:%s@{host}/{database}" + % quote_plus(password) + ) + except BaseException as error: + raise ValueError( + f"Unable to connect to SQL Database\n{error}" + ) from error - engine = create_engine( - f"{dialect}+{driver}://{user}:%s@{host}/{database}" % quote_plus(password) - ) - metadata_obj = MetaData() - metadata_obj.reflect(engine) - sql_database = SQLDatabase(engine, include_tables=tables) - - self.sqlalchemy_engine = engine - self.sql_database = sql_database - self.metadata_obj = metadata_obj + metadata_obj = MetaData() + metadata_obj.reflect(engine) + sql_database = SQLDatabase(engine, include_tables=tables) + self.sqlalchemy_engine = engine + self.sql_database = sql_database + self.metadata_obj = metadata_obj def get_nlsql_query_engine( self, diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index c78b8eea0..3244dc581 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -218,17 +218,21 @@ class QdrantSettings(BaseModel): class SQLDatabaseSettings(BaseModel): + enabled: bool = Field( + False, + description="Flag to enable SQL Query mode. Disabled by default", + ) dialect: Literal["mysql"] driver: Literal["pymysql"] - host: str | None = Field( + db_host: str | None = Field( "localhost", - description="Host name of Qdrant service. If host is None, set to 'localhost'.", + description="Host name of Database server. If host is None, set to 'localhost'.", ) - user: str | None = Field( + db_user: str | None = Field( "root", description="Username to be used for accessing the SQL Database Server. If user is None, set to 'root'.", ) - password: str = Field( + db_password: str = Field( "", description="Password to be used for accessing the SQL Database Server. If password is None, set to empty string.", ) @@ -249,7 +253,7 @@ class Settings(BaseModel): openai: OpenAISettings vectorstore: VectorstoreSettings qdrant: QdrantSettings | None = None - sqldatabase: SQLDatabaseSettings + sqlmode: SQLDatabaseSettings """ diff --git a/private_gpt/ui/ui.py b/private_gpt/ui/ui.py index 1efef0b35..8c2f50730 100644 --- a/private_gpt/ui/ui.py +++ b/private_gpt/ui/ui.py @@ -34,7 +34,9 @@ SOURCES_SEPARATOR = "\n\n Sources: \n" -MODES = ["Query Docs", "Query Db", "Search in Docs", "LLM Chat"] +MODES = ["Query Docs", "Search in Docs", "LLM Chat"] +if settings().sqlmode.enabled: + MODES.append("Query Db") class Source(BaseModel): diff --git a/pyproject.toml b/pyproject.toml index 6864938a5..e7edfbfd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ qdrant-client = "^1.6.9" chromadb = {version = "^0.4.13", optional = true} gradio = "4.4.1" pymysql = "^1.1.0" +cryptography = "^41.0.7" [tool.poetry.group.dev.dependencies] black = "^22" diff --git a/settings.yaml b/settings.yaml index 832549fe7..668228dc1 100644 --- a/settings.yaml +++ b/settings.yaml @@ -44,12 +44,13 @@ vectorstore: qdrant: path: local_data/private_gpt/qdrant -sqldatabase: +sqlmode: + enabled: false dialect: mysql driver: pymysql - host: ${HOSTNAME:localhost} - user: ${USERNAME:root} - password: ${PASSWORD:} + db_host: ${HOSTNAME:localhost} + db_user: ${USERNAME:root} + db_password: ${PASSWORD:} database: ${DATABASE:} tables: ["${TABLES:}"] From 87528296867f6b17c05c5861cfb03ddb18604e70 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 17:50:27 +0530 Subject: [PATCH 11/20] Fixes as per feedback: Update documentation, Variables, Dependency group --- fern/docs.yml | 4 +- fern/docs/pages/manual/contexts.mdx | 44 +++++++++++++++++++ fern/docs/pages/manual/nlsql.mdx | 30 ------------- .../embedding/embedding_component.py | 1 - .../components/nlsql/nlsql_component.py | 16 +++---- private_gpt/launcher.py | 1 - private_gpt/settings/settings.py | 34 ++++++++------ private_gpt/settings/yaml.py | 5 ++- private_gpt/ui/ui.py | 2 +- pyproject.toml | 9 ++-- scripts/ingest_folder.py | 1 - settings.yaml | 13 +++--- 12 files changed, 93 insertions(+), 67 deletions(-) create mode 100644 fern/docs/pages/manual/contexts.mdx delete mode 100644 fern/docs/pages/manual/nlsql.mdx diff --git a/fern/docs.yml b/fern/docs.yml index c77a474e1..3479cdec7 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -58,12 +58,12 @@ navigation: contents: - page: Vector Stores path: ./docs/pages/manual/vectordb.mdx - - page: SQL Databases - path: ./docs/pages/manual/nlsql.mdx - section: Advanced Setup contents: - page: LLM Backends path: ./docs/pages/manual/llms.mdx + - page: Context Configuration + path: ./docs/pages/manual/contexts.mdx - section: User Interface contents: - page: User interface (Gradio) Manual diff --git a/fern/docs/pages/manual/contexts.mdx b/fern/docs/pages/manual/contexts.mdx new file mode 100644 index 000000000..0b7769db8 --- /dev/null +++ b/fern/docs/pages/manual/contexts.mdx @@ -0,0 +1,44 @@ +## SQL Databases +Text-to-SQL querying has been tested for [MySQL](https://www.mysql.com/) in PrivateGPT. + +To use the text to SQL feature, set the following environment variables to allow connecting to your MySQL Server: + +```yaml +context_database: + db_dialect: mysql + db_driver: pymysql + db_host: + db_user: + db_password: + database: + tables: # eg: ["EMPLOYEES", "DEPARTMENTS"] +``` + +You will also need to install `pymysql`: `poetry add pymysql` + +## Other Databases +To start using other SQL databases as context, set the `context_database.db_dialect` and `context_database.db_driver` properties in the `settings.yaml` file. + +```yaml +sqldatabase: + dialect: + driver: +``` + +# List of Included Dialects (https://docs.sqlalchemy.org/en/20/dialects/#:~:text=driver%20is%20installed.-,Included%20Dialects%C2%B6,-PostgreSQL) + +| Database | Dialect | Driver Options | +|-----------------------|----------|---------------------------------------------------------- | +| PostgreSQL | psycopg2 | psycopg2/mysqlconnector/pg8000/asyncpg/ | +| MySQL | mysql | pymysql/mysqldb/mysqlconnector/asyncmy/aiomysql/cymysql | +| MariaDB | mariadb | pymysql/mysqldb/mariadbconnector/asyncmy/aiomysql/cymysql | +| SQLite | sqlite | pysqlite/aiosqlite/pysqlcipher | +| Oracle | oracle | cx_oracle/python-oracledb | +| Microsoft SQL Server | mssql | pyodbc/pymssql/aioodbc | + +# Here is an example for PostgreSQL: +Connection URL using psycopg2: `postgresql+psycopg2://scott:tiger@localhost/mydatabase` +Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.psycopg2:~:text=ON%20CONFLICT%20(Upsert)-,psycopg2,-%C2%B6 + + +# Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for constructing the DB connection string. \ No newline at end of file diff --git a/fern/docs/pages/manual/nlsql.mdx b/fern/docs/pages/manual/nlsql.mdx deleted file mode 100644 index e24c16be1..000000000 --- a/fern/docs/pages/manual/nlsql.mdx +++ /dev/null @@ -1,30 +0,0 @@ -## MySQL Configuration -Text-to-SQL querying has been tested for [MySQL](https://www.mysql.com/) in PrivateGPT. - -To use the text to SQL feature, set the following environment variables to allow connecting to your MySQL Server: - -```yaml -sqldatabase: - dialect: mysql - driver: pymysql - host: - user: - password: - database: - tables: -``` - -You will also need to install `pymysql`. - -`poetry add pymysql` - -## Other Databases -To get started with exploring the use of other databases, set the `sqldatabase.dialect` and `sqldatabase.driver` properties in the `settings.yaml` file. - -```yaml -sqldatabase: - dialect: - driver: -``` - -Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for constructing the DB connection string. \ No newline at end of file diff --git a/private_gpt/components/embedding/embedding_component.py b/private_gpt/components/embedding/embedding_component.py index e60c7af63..b48f87efd 100644 --- a/private_gpt/components/embedding/embedding_component.py +++ b/private_gpt/components/embedding/embedding_component.py @@ -27,7 +27,6 @@ def __init__(self, settings: Settings) -> None: cache_folder=str(models_cache_path), ) case "sagemaker": - from private_gpt.components.embedding.custom.sagemaker import ( SagemakerEmbedding, ) diff --git a/private_gpt/components/nlsql/nlsql_component.py b/private_gpt/components/nlsql/nlsql_component.py index b28854d45..c2d86b242 100644 --- a/private_gpt/components/nlsql/nlsql_component.py +++ b/private_gpt/components/nlsql/nlsql_component.py @@ -24,14 +24,14 @@ class NLSQLComponent: @inject def __init__(self, settings: Settings) -> None: - if settings.sqlmode.enabled: - dialect = settings.sqlmode.dialect - driver = settings.sqlmode.driver - host = settings.sqlmode.db_host - user = settings.sqlmode.db_user - password = settings.sqlmode.db_password - database = settings.sqlmode.database - tables = settings.sqlmode.tables + if settings.context_database.enabled: + dialect = settings.context_database.db_dialect + driver = settings.context_database.db_driver + host = settings.context_database.db_host + user = settings.context_database.db_user + password = settings.context_database.db_password + database = settings.context_database.database + tables = settings.context_database.tables try: engine = create_engine( f"{dialect}+{driver}://{user}:%s@{host}/{database}" diff --git a/private_gpt/launcher.py b/private_gpt/launcher.py index 791e841a6..6eb4d4540 100644 --- a/private_gpt/launcher.py +++ b/private_gpt/launcher.py @@ -17,7 +17,6 @@ def create_app(root_injector: Injector) -> FastAPI: - # Start the API async def bind_injector_to_request(request: Request) -> None: request.state.injector = root_injector diff --git a/private_gpt/settings/settings.py b/private_gpt/settings/settings.py index 3244dc581..850866807 100644 --- a/private_gpt/settings/settings.py +++ b/private_gpt/settings/settings.py @@ -222,24 +222,32 @@ class SQLDatabaseSettings(BaseModel): False, description="Flag to enable SQL Query mode. Disabled by default", ) - dialect: Literal["mysql"] - driver: Literal["pymysql"] - db_host: str | None = Field( + db_dialect: str = Field( + None, + description="Supported dialect in SQLAlchemy to be used for connecting with the DBAPI", + ) + db_driver: str = Field( + None, description="Drivername of the DBAPI for connecting with Database" + ) + db_host: str = Field( "localhost", - description="Host name of Database server. If host is None, set to 'localhost'.", + description="Host name of Database server. Defaults to 'localhost'.", ) - db_user: str | None = Field( - "root", - description="Username to be used for accessing the SQL Database Server. If user is None, set to 'root'.", + db_user: str = Field( + None, + description="Username to be used for accessing the SQL Database Server. Defaults to None.", ) db_password: str = Field( - "", - description="Password to be used for accessing the SQL Database Server. If password is None, set to empty string.", + None, + description="Password to be used for accessing the SQL Database Server. Defaults to None.", + ) + database: str = Field( + None, + description="The database name in which tables are to be queried. Defaults to None.", ) - database: str | None = Field( - description="The database name in which tables are to be queried", + tables: list[str] | None = Field( + None, description="List of tables to use as context. Defaults to [None]" ) - tables: list[str] | None = Field(description="List of tables to query into") class Settings(BaseModel): @@ -253,7 +261,7 @@ class Settings(BaseModel): openai: OpenAISettings vectorstore: VectorstoreSettings qdrant: QdrantSettings | None = None - sqlmode: SQLDatabaseSettings + context_database: SQLDatabaseSettings """ diff --git a/private_gpt/settings/yaml.py b/private_gpt/settings/yaml.py index a129ca495..77a250d75 100644 --- a/private_gpt/settings/yaml.py +++ b/private_gpt/settings/yaml.py @@ -25,7 +25,10 @@ def load_env_var(_, node) -> str: split = value.split(":", 1) env_var = split[0] value = environ.get(env_var) - default = None if len(split) == 1 else split[1] + if len(split) == 1: + default = None + else: + default = split[1].split(',') if env_var == "TABLES_LIST" else split[1] if value is None and default is None: raise ValueError( f"Environment variable {env_var} is not set and not default was provided" diff --git a/private_gpt/ui/ui.py b/private_gpt/ui/ui.py index 8c2f50730..6eaefe42a 100644 --- a/private_gpt/ui/ui.py +++ b/private_gpt/ui/ui.py @@ -35,7 +35,7 @@ SOURCES_SEPARATOR = "\n\n Sources: \n" MODES = ["Query Docs", "Search in Docs", "LLM Chat"] -if settings().sqlmode.enabled: +if settings().context_database.enabled: MODES.append("Query Db") diff --git a/pyproject.toml b/pyproject.toml index e7edfbfd7..dec4a557f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,9 +16,6 @@ llama-index = { extras = ["local_models"], version = "0.9.3" } watchdog = "^3.0.0" qdrant-client = "^1.6.9" chromadb = {version = "^0.4.13", optional = true} -gradio = "4.4.1" -pymysql = "^1.1.0" -cryptography = "^41.0.7" [tool.poetry.group.dev.dependencies] black = "^22" @@ -46,6 +43,12 @@ sentence-transformers = "^2.2.2" torch = ">=2.0.0, !=2.0.1, !=2.1.0" transformers = "^4.34.0" +# Dependencies for using Database as context +[tool.poetry.group.context_database] +optional = true +[tool.poetry.group.context_database.dependencies] +cryptography = "^41.0.7" + [tool.poetry.extras] chroma = ["chromadb"] diff --git a/scripts/ingest_folder.py b/scripts/ingest_folder.py index 37344ea55..5ff1fb218 100755 --- a/scripts/ingest_folder.py +++ b/scripts/ingest_folder.py @@ -84,7 +84,6 @@ def _do_ingest_one(self, changed_path: Path) -> None: logger.addHandler(file_handler) if __name__ == "__main__": - root_path = Path(args.folder) if not root_path.exists(): raise ValueError(f"Path {args.folder} does not exist") diff --git a/settings.yaml b/settings.yaml index 668228dc1..cd8a5dfd3 100644 --- a/settings.yaml +++ b/settings.yaml @@ -44,15 +44,16 @@ vectorstore: qdrant: path: local_data/private_gpt/qdrant -sqlmode: +context_database: + # Refer https://docs.privategpt.dev/manual/advanced-setup/context-configuration enabled: false - dialect: mysql - driver: pymysql - db_host: ${HOSTNAME:localhost} - db_user: ${USERNAME:root} + db_dialect: ${DIALECT:} + db_driver: ${DRIVER:} + db_host: ${HOSTNAME:} + db_user: ${USERNAME:} db_password: ${PASSWORD:} database: ${DATABASE:} - tables: ["${TABLES:}"] + tables: ${TABLES_LIST:} # Should be comma separated, for example: TABLE1,TABLE2,TABLE3 local: prompt_style: "llama2" From 965baf9a2e63417af40d07f6c4a318c4d1404b50 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 17:53:44 +0530 Subject: [PATCH 12/20] Fix docs --- fern/docs/pages/manual/contexts.mdx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fern/docs/pages/manual/contexts.mdx b/fern/docs/pages/manual/contexts.mdx index 0b7769db8..e6c1d80d9 100644 --- a/fern/docs/pages/manual/contexts.mdx +++ b/fern/docs/pages/manual/contexts.mdx @@ -25,7 +25,7 @@ sqldatabase: driver: ``` -# List of Included Dialects (https://docs.sqlalchemy.org/en/20/dialects/#:~:text=driver%20is%20installed.-,Included%20Dialects%C2%B6,-PostgreSQL) +# List of Included Dialects | Database | Dialect | Driver Options | |-----------------------|----------|---------------------------------------------------------- | @@ -36,9 +36,11 @@ sqldatabase: | Oracle | oracle | cx_oracle/python-oracledb | | Microsoft SQL Server | mssql | pyodbc/pymssql/aioodbc | +https://docs.sqlalchemy.org/en/20/dialects/ + # Here is an example for PostgreSQL: Connection URL using psycopg2: `postgresql+psycopg2://scott:tiger@localhost/mydatabase` -Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.psycopg2:~:text=ON%20CONFLICT%20(Upsert)-,psycopg2,-%C2%B6 +Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html # Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for constructing the DB connection string. \ No newline at end of file From cdabad0b745aae13328fdc8a5e176dfe1d7999f9 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 18:02:50 +0530 Subject: [PATCH 13/20] More fixes --- fern/docs/pages/manual/contexts.mdx | 4 ++-- private_gpt/settings/yaml.py | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/fern/docs/pages/manual/contexts.mdx b/fern/docs/pages/manual/contexts.mdx index e6c1d80d9..198d5f6cd 100644 --- a/fern/docs/pages/manual/contexts.mdx +++ b/fern/docs/pages/manual/contexts.mdx @@ -11,7 +11,7 @@ context_database: db_user: db_password: database: - tables: # eg: ["EMPLOYEES", "DEPARTMENTS"] + tables: # eg: TABLE1,TABLE2,TABLE3 ``` You will also need to install `pymysql`: `poetry add pymysql` @@ -43,4 +43,4 @@ Connection URL using psycopg2: `postgresql+psycopg2://scott:tiger@localhost/myda Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html -# Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for constructing the DB connection string. \ No newline at end of file +Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for constructing the DB connection string. \ No newline at end of file diff --git a/private_gpt/settings/yaml.py b/private_gpt/settings/yaml.py index 77a250d75..9de19d130 100644 --- a/private_gpt/settings/yaml.py +++ b/private_gpt/settings/yaml.py @@ -25,10 +25,9 @@ def load_env_var(_, node) -> str: split = value.split(":", 1) env_var = split[0] value = environ.get(env_var) - if len(split) == 1: - default = None - else: - default = split[1].split(',') if env_var == "TABLES_LIST" else split[1] + if env_var == "TABLES_LIST": + value = value.split(",") + default = None if len(split) == 1 else split[1] if value is None and default is None: raise ValueError( f"Environment variable {env_var} is not set and not default was provided" From 9de7f3e459e3d18d526f1314251e5326b52bbbe7 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 18:29:09 +0530 Subject: [PATCH 14/20] Final fixes --- fern/docs/pages/manual/contexts.mdx | 8 +++++--- private_gpt/settings/yaml.py | 7 +++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/fern/docs/pages/manual/contexts.mdx b/fern/docs/pages/manual/contexts.mdx index 198d5f6cd..694c919e1 100644 --- a/fern/docs/pages/manual/contexts.mdx +++ b/fern/docs/pages/manual/contexts.mdx @@ -25,7 +25,7 @@ sqldatabase: driver: ``` -# List of Included Dialects +## List of Included Dialects in SQLAlchemy | Database | Dialect | Driver Options | |-----------------------|----------|---------------------------------------------------------- | @@ -38,8 +38,10 @@ sqldatabase: https://docs.sqlalchemy.org/en/20/dialects/ -# Here is an example for PostgreSQL: -Connection URL using psycopg2: `postgresql+psycopg2://scott:tiger@localhost/mydatabase` +## For PostgreSQL: +Sample connection URL (using psycopg2): `postgresql+psycopg2://scott:tiger@localhost/mydatabase` + + Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html diff --git a/private_gpt/settings/yaml.py b/private_gpt/settings/yaml.py index 9de19d130..093db2a3b 100644 --- a/private_gpt/settings/yaml.py +++ b/private_gpt/settings/yaml.py @@ -25,9 +25,12 @@ def load_env_var(_, node) -> str: split = value.split(":", 1) env_var = split[0] value = environ.get(env_var) - if env_var == "TABLES_LIST": - value = value.split(",") default = None if len(split) == 1 else split[1] + if env_var in ["TABLES_LIST"]: + if value is not None: + value = value.split(",") + else: + default = [] if value is None and default is None: raise ValueError( f"Environment variable {env_var} is not set and not default was provided" From fd04e173c3a2cf6a9c84b3d89c0dc8bce44411af Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 18:34:47 +0530 Subject: [PATCH 15/20] conditionally import sqlalchemy --- .../components/nlsql/nlsql_component.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/private_gpt/components/nlsql/nlsql_component.py b/private_gpt/components/nlsql/nlsql_component.py index c2d86b242..34d7c9e2a 100644 --- a/private_gpt/components/nlsql/nlsql_component.py +++ b/private_gpt/components/nlsql/nlsql_component.py @@ -1,15 +1,11 @@ import logging +from typing import Any from urllib.parse import quote_plus from injector import inject, singleton from llama_index import ServiceContext, SQLDatabase, VectorStoreIndex from llama_index.indices.struct_store import SQLTableRetrieverQueryEngine from llama_index.objects import ObjectIndex, SQLTableNodeMapping, SQLTableSchema -from sqlalchemy import ( - MetaData, -) -from sqlalchemy.engine import create_engine -from sqlalchemy.engine.base import Engine from private_gpt.settings.settings import Settings @@ -18,9 +14,9 @@ @singleton class NLSQLComponent: - sqlalchemy_engine: Engine - sql_database: SQLDatabase - metadata_obj: MetaData + sqlalchemy_engine: Any + sql_database: Any + metadata_obj: Any @inject def __init__(self, settings: Settings) -> None: @@ -33,13 +29,18 @@ def __init__(self, settings: Settings) -> None: database = settings.context_database.database tables = settings.context_database.tables try: + from sqlalchemy import ( + MetaData, + ) + from sqlalchemy.engine import create_engine + engine = create_engine( f"{dialect}+{driver}://{user}:%s@{host}/{database}" % quote_plus(password) ) except BaseException as error: raise ValueError( - f"Unable to connect to SQL Database\n{error}" + f"Unable to initialise connection to SQL Database\n{error}" ) from error metadata_obj = MetaData() From d805c98e0396c239216f3c7ea3339a74b3c7d18f Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 19:08:41 +0530 Subject: [PATCH 16/20] Make docs user friendly --- fern/docs/pages/manual/contexts.mdx | 41 ++++++++++++++++++----------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/fern/docs/pages/manual/contexts.mdx b/fern/docs/pages/manual/contexts.mdx index 694c919e1..2567a9126 100644 --- a/fern/docs/pages/manual/contexts.mdx +++ b/fern/docs/pages/manual/contexts.mdx @@ -1,12 +1,30 @@ ## SQL Databases Text-to-SQL querying has been tested for [MySQL](https://www.mysql.com/) in PrivateGPT. -To use the text to SQL feature, set the following environment variables to allow connecting to your MySQL Server: +The DB connection is established using [SQLAlchemy](https://www.sqlalchemy.org/), and allows connecting to a number of databases. +To use this feature, connection details for the DB are required. +A database name and list of tables allow PrivateGPT to select those as the query context. + +##### Install Dependencies +* Run the following command to install the dependencies for text-to-SQL: + +```bash + poetry install --with context_database +``` + +* Install an appropriate database driver. For MySQL as an example, run: + +```bash + poetry add pymysql +``` + +* Set the following environment variables in `settings.yaml`: ```yaml context_database: - db_dialect: mysql - db_driver: pymysql + enabled: true + db_dialect: + db_driver: db_host: db_user: db_password: @@ -14,18 +32,11 @@ context_database: tables: # eg: TABLE1,TABLE2,TABLE3 ``` -You will also need to install `pymysql`: `poetry add pymysql` - -## Other Databases +##### Other Databases To start using other SQL databases as context, set the `context_database.db_dialect` and `context_database.db_driver` properties in the `settings.yaml` file. -```yaml -sqldatabase: - dialect: - driver: -``` -## List of Included Dialects in SQLAlchemy +##### List of Included Dialects in SQLAlchemy | Database | Dialect | Driver Options | |-----------------------|----------|---------------------------------------------------------- | @@ -38,11 +49,11 @@ sqldatabase: https://docs.sqlalchemy.org/en/20/dialects/ -## For PostgreSQL: +##### Example For PostgreSQL: Sample connection URL (using psycopg2): `postgresql+psycopg2://scott:tiger@localhost/mydatabase` -Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html +PostgreSQL Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html -Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for constructing the DB connection string. \ No newline at end of file +Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for documentation about configuring SQLAlchemy and the DB connection string. \ No newline at end of file From cb1ecf0606ec776f30012e88bbe02b4730312e73 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 19:30:19 +0530 Subject: [PATCH 17/20] Add steps, more details in docs --- fern/docs/pages/manual/contexts.mdx | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/fern/docs/pages/manual/contexts.mdx b/fern/docs/pages/manual/contexts.mdx index 2567a9126..8d78d19eb 100644 --- a/fern/docs/pages/manual/contexts.mdx +++ b/fern/docs/pages/manual/contexts.mdx @@ -5,7 +5,7 @@ The DB connection is established using [SQLAlchemy](https://www.sqlalchemy.org/) To use this feature, connection details for the DB are required. A database name and list of tables allow PrivateGPT to select those as the query context. -##### Install Dependencies +### Install Dependencies * Run the following command to install the dependencies for text-to-SQL: ```bash @@ -32,9 +32,16 @@ context_database: tables: # eg: TABLE1,TABLE2,TABLE3 ``` -##### Other Databases +### Other Databases +* Dialect is the system SQLAlchemy uses to communicate with different databases. +* Drivers are the connectors for databases. + To start using other SQL databases as context, set the `context_database.db_dialect` and `context_database.db_driver` properties in the `settings.yaml` file. +This configures a connection string as per the following format: +```bash + db_dialect+db_driver://db_user:db_password@db_host/database +``` ##### List of Included Dialects in SQLAlchemy @@ -47,13 +54,20 @@ To start using other SQL databases as context, set the `context_database.db_dial | Oracle | oracle | cx_oracle/python-oracledb | | Microsoft SQL Server | mssql | pyodbc/pymssql/aioodbc | -https://docs.sqlalchemy.org/en/20/dialects/ -##### Example For PostgreSQL: -Sample connection URL (using psycopg2): `postgresql+psycopg2://scott:tiger@localhost/mydatabase` +##### Examples Connection Strings + +| Database | Example | +|-----------------------|----------| +| MySQL | `mysql+pymysql://:@/` | +| MariaDB | `mariadb+mariadbconnector://:@[:]/` | +| PostgreSQL | `postgresql+psycopg2://user:password@host:port/dbname` | +| SQLite | `sqlite+pysqlite:///file_path` | -PostgreSQL Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html +##### Additional Documentation +Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for documentation about configuring SQLAlchemy and the DB connection string. +List of Dialects: https://docs.sqlalchemy.org/en/20/dialects/ -Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for documentation about configuring SQLAlchemy and the DB connection string. \ No newline at end of file +PostgreSQL Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html \ No newline at end of file From 5ef61b44457e597dc4c6574fcd4b0698fde112ac Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 19:47:30 +0530 Subject: [PATCH 18/20] fix typos --- fern/docs.yml | 2 +- fern/docs/pages/manual/contexts.mdx | 30 ++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/fern/docs.yml b/fern/docs.yml index 3479cdec7..040c54726 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -62,7 +62,7 @@ navigation: contents: - page: LLM Backends path: ./docs/pages/manual/llms.mdx - - page: Context Configuration + - page: Context Configurations path: ./docs/pages/manual/contexts.mdx - section: User Interface contents: diff --git a/fern/docs/pages/manual/contexts.mdx b/fern/docs/pages/manual/contexts.mdx index 8d78d19eb..16ebe5376 100644 --- a/fern/docs/pages/manual/contexts.mdx +++ b/fern/docs/pages/manual/contexts.mdx @@ -12,10 +12,10 @@ A database name and list of tables allow PrivateGPT to select those as the query poetry install --with context_database ``` -* Install an appropriate database driver. For MySQL as an example, run: +* Install an appropriate database driver: ```bash - poetry add pymysql + poetry add ``` * Set the following environment variables in `settings.yaml`: @@ -32,7 +32,7 @@ context_database: tables: # eg: TABLE1,TABLE2,TABLE3 ``` -### Other Databases +### Configuring Database * Dialect is the system SQLAlchemy uses to communicate with different databases. * Drivers are the connectors for databases. @@ -43,19 +43,19 @@ This configures a connection string as per the following format: db_dialect+db_driver://db_user:db_password@db_host/database ``` -##### List of Included Dialects in SQLAlchemy +##### List of included Dialects in SQLAlchemy -| Database | Dialect | Driver Options | -|-----------------------|----------|---------------------------------------------------------- | -| PostgreSQL | psycopg2 | psycopg2/mysqlconnector/pg8000/asyncpg/ | -| MySQL | mysql | pymysql/mysqldb/mysqlconnector/asyncmy/aiomysql/cymysql | -| MariaDB | mariadb | pymysql/mysqldb/mariadbconnector/asyncmy/aiomysql/cymysql | -| SQLite | sqlite | pysqlite/aiosqlite/pysqlcipher | -| Oracle | oracle | cx_oracle/python-oracledb | -| Microsoft SQL Server | mssql | pyodbc/pymssql/aioodbc | +| Database | Dialect | Driver Options | +|-----------------------|------------|---------------------------------------------------------- | +| PostgreSQL | postgresql | psycopg2/mysqlconnector/pg8000/asyncpg/ | +| MySQL | mysql | pymysql/mysqldb/mysqlconnector/asyncmy/aiomysql/cymysql | +| MariaDB | mariadb | pymysql/mysqldb/mariadbconnector/asyncmy/aiomysql/cymysql | +| SQLite | sqlite | pysqlite/aiosqlite/pysqlcipher | +| Oracle | oracle | cx_oracle/python-oracledb | +| Microsoft SQL Server | mssql | pyodbc/pymssql/aioodbc | -##### Examples Connection Strings +##### Example connection strings | Database | Example | |-----------------------|----------| @@ -65,9 +65,9 @@ This configures a connection string as per the following format: | SQLite | `sqlite+pysqlite:///file_path` | -##### Additional Documentation +##### Additional documentation Refer to [SQLAlchemy Engine Configuration](https://docs.sqlalchemy.org/en/20/core/engines.html) for documentation about configuring SQLAlchemy and the DB connection string. -List of Dialects: https://docs.sqlalchemy.org/en/20/dialects/ +List of internal & external Dialects: https://docs.sqlalchemy.org/en/20/dialects/ PostgreSQL Documentation Reference: https://docs.sqlalchemy.org/en/20/dialects/postgresql.html \ No newline at end of file From 3fd36d2d4b8ad89525c2c8756d19e2214a4ad89a Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 20:26:25 +0530 Subject: [PATCH 19/20] Don't update poetry.lock --- poetry.lock | 162 +++++--------------------------------------------- settings.yaml | 2 +- 2 files changed, 15 insertions(+), 149 deletions(-) diff --git a/poetry.lock b/poetry.lock index ddf5924f9..4954fb014 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. [[package]] name = "accelerate" @@ -391,70 +391,6 @@ files = [ {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] -[[package]] -name = "cffi" -version = "1.16.0" -description = "Foreign Function Interface for Python calling C code." -optional = false -python-versions = ">=3.8" -files = [ - {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, - {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, - {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, - {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, - {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, - {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, - {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, - {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, - {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, - {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, - {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, - {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, - {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, - {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, - {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, -] - -[package.dependencies] -pycparser = "*" - [[package]] name = "cfgv" version = "3.4.0" @@ -808,51 +744,6 @@ files = [ [package.extras] toml = ["tomli"] -[[package]] -name = "cryptography" -version = "41.0.7" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -optional = false -python-versions = ">=3.7" -files = [ - {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, - {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, - {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, - {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, - {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, - {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, - {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, - {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, - {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, - {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, - {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, -] - -[package.dependencies] -cffi = ">=1.12" - -[package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -nox = ["nox"] -pep8test = ["black", "check-sdist", "mypy", "ruff"] -sdist = ["build"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] -test-randomorder = ["pytest-randomly"] - [[package]] name = "cycler" version = "0.12.1" @@ -3364,25 +3255,22 @@ virtualenv = ">=20.10.0" [[package]] name = "protobuf" -version = "4.21.12" +version = "4.25.1" description = "" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "protobuf-4.21.12-cp310-abi3-win32.whl", hash = "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1"}, - {file = "protobuf-4.21.12-cp310-abi3-win_amd64.whl", hash = "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2"}, - {file = "protobuf-4.21.12-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791"}, - {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97"}, - {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7"}, - {file = "protobuf-4.21.12-cp37-cp37m-win32.whl", hash = "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717"}, - {file = "protobuf-4.21.12-cp37-cp37m-win_amd64.whl", hash = "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574"}, - {file = "protobuf-4.21.12-cp38-cp38-win32.whl", hash = "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec"}, - {file = "protobuf-4.21.12-cp38-cp38-win_amd64.whl", hash = "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30"}, - {file = "protobuf-4.21.12-cp39-cp39-win32.whl", hash = "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc"}, - {file = "protobuf-4.21.12-cp39-cp39-win_amd64.whl", hash = "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b"}, - {file = "protobuf-4.21.12-py2.py3-none-any.whl", hash = "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5"}, - {file = "protobuf-4.21.12-py3-none-any.whl", hash = "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462"}, - {file = "protobuf-4.21.12.tar.gz", hash = "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab"}, + {file = "protobuf-4.25.1-cp310-abi3-win32.whl", hash = "sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7"}, + {file = "protobuf-4.25.1-cp310-abi3-win_amd64.whl", hash = "sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b"}, + {file = "protobuf-4.25.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd"}, + {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb"}, + {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7"}, + {file = "protobuf-4.25.1-cp38-cp38-win32.whl", hash = "sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd"}, + {file = "protobuf-4.25.1-cp38-cp38-win_amd64.whl", hash = "sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0"}, + {file = "protobuf-4.25.1-cp39-cp39-win32.whl", hash = "sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510"}, + {file = "protobuf-4.25.1-cp39-cp39-win_amd64.whl", hash = "sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10"}, + {file = "protobuf-4.25.1-py3-none-any.whl", hash = "sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6"}, + {file = "protobuf-4.25.1.tar.gz", hash = "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2"}, ] [[package]] @@ -3544,17 +3432,6 @@ files = [ [package.dependencies] pyasn1 = ">=0.4.6,<0.6.0" -[[package]] -name = "pycparser" -version = "2.21" -description = "C parser in Python" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, - {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, -] - [[package]] name = "pydantic" version = "2.5.1" @@ -3749,17 +3626,6 @@ files = [ plugins = ["importlib-metadata"] windows-terminal = ["colorama (>=0.4.6)"] -[[package]] -name = "pymysql" -version = "1.1.0" -description = "Pure Python MySQL Driver" -optional = false -python-versions = ">=3.7" -files = [ - {file = "PyMySQL-1.1.0-py3-none-any.whl", hash = "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7"}, - {file = "PyMySQL-1.1.0.tar.gz", hash = "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96"}, -] - [package.extras] ed25519 = ["PyNaCl (>=1.4.0)"] rsa = ["cryptography"] diff --git a/settings.yaml b/settings.yaml index cd8a5dfd3..d028fe27c 100644 --- a/settings.yaml +++ b/settings.yaml @@ -49,7 +49,7 @@ context_database: enabled: false db_dialect: ${DIALECT:} db_driver: ${DRIVER:} - db_host: ${HOSTNAME:} + db_host: ${HOSTNAME:localhost} db_user: ${USERNAME:} db_password: ${PASSWORD:} database: ${DATABASE:} From 8e931be618826ea403fc68839e8390c16d60e683 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Sun, 17 Dec 2023 20:27:07 +0530 Subject: [PATCH 20/20] Don't update poetry.lock --- poetry.lock | 4 ---- 1 file changed, 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4954fb014..6ce5e62d7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3626,10 +3626,6 @@ files = [ plugins = ["importlib-metadata"] windows-terminal = ["colorama (>=0.4.6)"] -[package.extras] -ed25519 = ["PyNaCl (>=1.4.0)"] -rsa = ["cryptography"] - [[package]] name = "pyparsing" version = "3.1.1"