diff --git a/.gitignore b/.gitignore index 566f9a4f5..1976bee0f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,14 @@ dist **/*.lic .vscode .idea +.env .editorconfig .DS_Store **/*.pyc ./berkeley-function-call-leaderboard/function_credential_config.json ./berkeley-function-call-leaderboard/eval_checker/tree-sitter-java -./berkeley-function-call-leaderboard/eval_checker/tree-sitter-javascript \ No newline at end of file +./berkeley-function-call-leaderboard/eval_checker/tree-sitter-javascript +goex/exec_engine/checkpoints/ +goex/exec_engine/credentials/secret_store.json +goex/docker/*/requirements.txt +goex/docker/misc/images.json diff --git a/README.md b/README.md index 0e426f80c..1fb555489 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ +**🚒 GoEx: A Runtime for executing LLM generated actions like code & API calls** GoEx presents “undo” and “damage confinement” abstractions for mitigating the risk of unintended actions taken in LLM-powered systems. [Release blog](https://gorilla.cs.berkeley.edu/blogs/10_gorilla_exec_engine.html) [Paper](https://arxiv.org/abs/2404.06921). + **🎉 Berkeley Function Calling Leaderboard** How do models stack up for function calling? :dart: Releasing the [Berkeley Function Calling Leaderboard](https://gorilla.cs.berkeley.edu/leaderboard). Read more in our [Release Blog](https://gorilla.cs.berkeley.edu/blogs/8_berkeley_function_calling_leaderboard.html). **:trophy: Gorilla OpenFunctions v2** Sets new SoTA for open-source LLMs :muscle: On-par with GPT-4 :raised_hands: Supports more languages :ok_hand: [Blog](https://gorilla.cs.berkeley.edu/blogs/7_open_functions_v2.html). diff --git a/goex/.env.example b/goex/.env.example new file mode 100644 index 000000000..331aae516 --- /dev/null +++ b/goex/.env.example @@ -0,0 +1,8 @@ +# mysql +DATABASE_USER=root +DATABASE_PASSWORD=abcdefg +DATABASE_HOST=localhost +DATABASE_NAME=testdb + +# sqlite +DATABASE_PATH=goex_demo.db diff --git a/goex/CONTRIBUTE.md b/goex/CONTRIBUTE.md new file mode 100644 index 000000000..3bbfc0589 --- /dev/null +++ b/goex/CONTRIBUTE.md @@ -0,0 +1,16 @@ +# Contributing to the Repo + +## Adding a new database + +Adding a new database will require the following to be added: + +1. A new Manager subclass (like the DBManager classes in `exec_engine/db_manager.py`) + 1. Needs to include logic for supporting transactional commits/rollbacks regardless of the database type (Ensure reversibility for all database types) + 2. Implementation should be similar to ` MySQLManager` or `SQLiteManager` + 3. Be able to format a user prompt with additional database specific info (like schema) to inform the LLM how to generate an accurate API call. +2. \[Optional\] If dry-running is wanted: + 1. Add a new folder inside `docker/` much like `docker/mysql_docker` + 2. Make sure the dockerfile sets up the environment for the database deployment + 3. Create a copy of the database and copy it to the docker container + 4. Create a reversibility script for dry-running like `exec_engine/db_reversion_test.txt` + 1. This script is written in txt since it will be passed into the docker environment and run with `python_executor.py` diff --git a/goex/README.md b/goex/README.md new file mode 100644 index 000000000..7c675c59c --- /dev/null +++ b/goex/README.md @@ -0,0 +1,241 @@ +# 🦍 GoEx + +| [**Website**](https://goex.gorilla-llm.com/index) | + +GoEx provides a new way to interact with your favorite APIs! Powered by LLMs, GoEx executes generated code, taking into consideration any credentials you may need to access APIs. Execution security is ensured by creating reverse calls that undo any unwanted effects. Ask GoEx for anything! + +## Contents + +- [Install](#install) +- [CLI Usage](#cli-usage) + - [RESTful API](#restful-api) + - [Database](#database) + - [Filesystem](#filesystem) +- [Credentials \& Authorization Token](#credentials--authorization-token) + +## Install + +1.) Navigate inside the goex folder and set up a clean environment with **Conda** or **Python venv** + +```sh +python3 -m venv goex-env +source goex-env/bin/activate +``` + +**OR** + +```sh +conda create --name goex python=3.10 -y +conda activate goex +``` + +2.) Install the `goex` CLI tool + +```sh +pip install -e . +``` + +3.) Install and initialize Mkcert to support OAuth2 token exchange [required for services that require https for redirect URL e.g Slack] + +Mac: + +```sh +brew install mkcert +mkcert -install +mkcert localhost +``` + +Windows: + +```sh +choco install mkcert +mkcert -install +mkcert localhost +``` + +Mkcert creates a local certificate authority, enabling Gorilla to establish secure communication for OAuth2 token exchange between localhost and interacting services. The command may prompt the user to pass in their password. + +## CLI Usage + +List all commands Gorilla currently supports and their usage: + +```sh +goex -h +``` + +### RESTful API + +Give authorizations and perform OAuth2 token exchanges with services Gorilla currently support. + +```sh +goex -authorize # gmail, slack, spotify, github, dropbox +``` + +After a service is authorized, user will be able to interact with it by providing a prompt. + +```sh +# example 1 +# please first run goex -authorize slack +goex execute -prompt send a funny joke to the user with email gorilla@yahoo.com on slack -type rest +``` + +```sh +# example 2 +# this action needs user to authorize gmail +goex execute -prompt who are the senders of the 5 latest emails in my gmail inbox -type rest +``` + +**[Beta]** User can also specify `-generate_mode function_in_context` to generate API calls with function calling. Functions are stored in the **functions** folder, and it currently only supports a limited number of Slack features. + +```sh +#example 3 (function calling) +goex execute -prompt add a smile emoji to the latest message sent to channel on slack -type rest -generate_mode function_in_context +``` + +List all commands Gorilla current supports and their usages + +### Database + +To test out database interactions locally, each database server requires its own setup + +#### SQLite + +- If you need to create a new SQLite DB, do the following: + + ``` + sqlite3 ./goex_demo.db + ``` + + Then after, you will need to use the sqlite CLI: + + ``` + sqlite> .database + ``` + + Press `Ctrl-D` to exit out and now your `.db` file will be created! + +- Run `demo/env_setup.py` to get your environment variables set up + Use default values if you are running just the demo. + + ``` + python3 demo/env_setup.py + ``` + +- Set GoEx to use SQLite for DB operations + ``` + goex -set_config dbtype sqlite + ``` + +#### Try it out! + +After setting up your SQL database, you can try the following examples! + +``` +goex execute -prompt "Create a new table for storing user information called users, with each user having a current balance column denoting their bank balance" -type db +``` + +``` +goex execute -prompt "Add 3 example users to the users table" -type db +``` + +``` +goex execute -prompt "In the users table, add 500 to everyone's current balance" -type db +``` + +#### MySQL + +- Install MySQL Server + + - For non-Mac, [install server here](https://dev.mysql.com/downloads/mysql/) + - **Make sure to add `mysql` to path!** + - Mac: + + ``` + brew install mysql + ``` + +- If you don't have your own server, import the example database using `demo/mysql_setup.py` by running: + ```sh + cd demo + python3 mysql_setup.py testdb + ``` +- Put the user, password, host, and database name info into `.env` by running this script + + ``` + python3 demo/env_setup.py + ``` + +- Set GoEx to use MySQL for DB operations + ``` + goex -set_config dbtype mysql + ``` + +### Filesystem + +The goex command will be executed at the path pointed to by `fs_path` in `user_config.json`. If `fs_path` is not specified or is equal to `""`, execution will occur in the user's current working directory. Below is an example of how to set this up. + +Make sure to have Git LFS installed on your machine! + +Mac: `brew install git-lfs` +Windows/Linux: git-lfs.com and download + +#### Try it out! + +Let's first create a testing folder. + +``` +mkdir test +goex -set_config fs_path test +``` + +##### Create a Simple File + +``` +goex execute -prompt "Write a witty LLM joke into joke.txt" -type fs +``` + +##### Code Writing + +You can tell Gorilla-Ex to write code for you, and directly have it be written onto your chosen directory! + +``` +goex execute -prompt "Write a Python program that is a calculator into a file called calculator.py" -type fs +``` + +##### Log Compilation + +``` +goex execute -prompt "Create 5 log files named according to some fake dates and insert some placeholder log content into them" -type fs +``` + +Here you will have 5 log files with some placeholder content. You can execute the command below to compile the log file contents into a single CSV for easy analysis. + +``` +goex execute -prompt "Compile the log file contents into a single CSV with the date as the identifying column here." -type fs +``` + +## Credentials & Authorization Token + +There are two types of credentials the user can add to the execution engine. + +1.) [**OAuth 2.0**](https://oauth.net/2/) + +Gorilla-ex follows the standard OAuth 2.0 token exchanges flow. Running the command below will redirect the user to the browser, where they will be prompted to grant permissions to Gorilla for a specific set of scopes. + +```sh +goex -authorize +``` + +After Gorilla-ex receives the token for a service, it will automatically be able to recognize the keyword in future prompts, enabling the user to perform actions on that particular platform. Additionally, the token will not be exposed to the LLM and will only be visible during execution. + +We continually seek to expand our support for additional services. The authorization logic for each service resides in the authorization/scripts folder. We warmly welcome interested contributors to submit a pull request introducing a new authorization flow. **Your contributions are highly appreciated 🚀** + +2.) **Raw API Key** + +If the user wants to add services not supported by OAuth2, they can do so by calling the function below with `service` and `key` as parameters: + +```sh +goex -insert_creds alpha_vantage {API_KEY} +``` + +The key will be stored in Gorilla-ex's secret store and passed on to the LLM during prompting. diff --git a/goex/__init__.py b/goex/__init__.py new file mode 100644 index 000000000..401da57f8 --- /dev/null +++ b/goex/__init__.py @@ -0,0 +1 @@ +from .cli import main diff --git a/goex/authorizations/scripts/authorization_utils.py b/goex/authorizations/scripts/authorization_utils.py new file mode 100644 index 000000000..6dce46c1a --- /dev/null +++ b/goex/authorizations/scripts/authorization_utils.py @@ -0,0 +1,25 @@ + +import os +from pathlib import Path +import sys +import inspect + +currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parentdir = os.path.dirname(currentdir) +sys.path.insert(0, os.path.dirname(parentdir)) + +from exec_engine.credentials.credentials_utils import CREDS_FOLDER_PATH, insert_creds + +AUTHORIZATION_FOLDER_PATH = os.path.dirname(Path(os.path.realpath(__file__))) + +def authorize_service(name: str): + name = name.lower() + authorization_file = "{name}_authorization.py".format(name = name) + authorization_path = os.path.join(AUTHORIZATION_FOLDER_PATH, authorization_file) + + if not os.path.exists(authorization_path): + return False + else: + print(exec(open(authorization_path).read(), globals())) + insert_creds(name, os.path.join(CREDS_FOLDER_PATH, name), target = CREDS_FOLDER_PATH, cred_type="path") + return True diff --git a/goex/authorizations/scripts/consts.py b/goex/authorizations/scripts/consts.py new file mode 100644 index 000000000..af7b42bd5 --- /dev/null +++ b/goex/authorizations/scripts/consts.py @@ -0,0 +1,7 @@ +import os +from pathlib import Path + +AUTH_URL = "https://goex-services.gorilla-llm.com/authorize" +# AUTH_URL = "http://localhost:443/authorize" +CERT_FILE_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), "localhost.pem") +KEY_FILE_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), "localhost-key.pem") \ No newline at end of file diff --git a/goex/authorizations/scripts/discord_authorization.py b/goex/authorizations/scripts/discord_authorization.py new file mode 100644 index 000000000..96456d457 --- /dev/null +++ b/goex/authorizations/scripts/discord_authorization.py @@ -0,0 +1,98 @@ +from google_auth_oauthlib.flow import InstalledAppFlow, _WSGIRequestHandler, _RedirectWSGIApp +from google.auth.transport.requests import Request +import os +from pathlib import Path +import pickle +import re +import requests +import ssl +import webbrowser +import wsgiref.simple_server +import wsgiref.util +from urllib.parse import quote + +DISCORD_FOLDER_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), 'exec_engine/credentials/discord') +CREDENTIAL_FILE = os.path.join(os.path.dirname(Path(__file__).parent), "credentials/discord_credentials.json") +TOKEN_FILE = os.path.join(DISCORD_FOLDER_PATH, 'token.pickle') + +CERT_FILE_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), "localhost.pem") +KEY_FILE_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), "localhost-key.pem") + +def main(): + creds = None + # Check if token.pickle file exists and is valid + flow = InstalledAppFlow.from_client_secrets_file( + CREDENTIAL_FILE, []) + + creds = run_discord_flow(flow) + # Save the credentials for the next run + if not os.path.isdir(DISCORD_FOLDER_PATH): + os.mkdir(DISCORD_FOLDER_PATH) + with open(TOKEN_FILE, 'wb') as token: + pickle.dump(creds, token) + +def run_discord_flow(flow: InstalledAppFlow): + host="localhost" + bind_addr=None + port=8000 + authorization_prompt_message=flow._DEFAULT_AUTH_PROMPT_MESSAGE + success_message=flow._DEFAULT_WEB_SUCCESS_MESSAGE + open_browser=True, + redirect_uri_trailing_slash=True + timeout_seconds=None + browser=None + + wsgi_app = _RedirectWSGIApp(success_message) + # Fail fast if the address is occupied + wsgiref.simple_server.WSGIServer.allow_reuse_address = False + local_server = wsgiref.simple_server.make_server( + bind_addr or host, port, wsgi_app, handler_class=_WSGIRequestHandler + ) + + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslctx.check_hostname = False # If set to True, only the hostname that matches the certificate will be accepted + sslctx.load_cert_chain(certfile=CERT_FILE_PATH, keyfile=KEY_FILE_PATH) + + local_server.socket = ssl.wrap_socket (local_server.socket, + keyfile=KEY_FILE_PATH, + certfile=CERT_FILE_PATH, server_side=True) + + redirect_uri_format = ( + "https://{}:{}/" if redirect_uri_trailing_slash else "https://{}:{}" + ) + redirect_uri = redirect_uri_format.format(host, local_server.server_port) + auth_url, _ = flow.authorization_url() + + if open_browser: + # if browser is None it defaults to default browser + webbrowser.get(browser).open(auth_url, new=1, autoraise=True) + + if authorization_prompt_message: + print(authorization_prompt_message.format(url=auth_url)) + + local_server.timeout = timeout_seconds + local_server.handle_request() + + # Note: using https here because oauthlib is very picky that + # OAuth 2.0 should only occur over https. + authorization_response = wsgi_app.last_request_uri.replace("http", "https") + print(authorization_response) + response_regex = r".*code=(.*)&state=(.*)" + print(re.findall(response_regex, authorization_response)) + code = re.findall(response_regex, authorization_response)[0] + + local_server.server_close() + + data = {"grant_type":"authorization_code", "code": code, "redirect_uri":redirect_uri} + + url = "https://discord.com/api/oauth2/token" + headers = {"Content-Type": "application/x-www-form-urlencoded"} + r = requests.post(url, + data=data, + headers=headers, + auth=("1214447388354420736", "Io25U2nXzMDE6GgtfBqmVWbLPhv7LuoG")) + + print(r.json()) + return r.json()['access_token'] + +main() diff --git a/goex/authorizations/scripts/dropbox_authorization.py b/goex/authorizations/scripts/dropbox_authorization.py new file mode 100644 index 000000000..85ca867f2 --- /dev/null +++ b/goex/authorizations/scripts/dropbox_authorization.py @@ -0,0 +1,83 @@ +from google_auth_oauthlib.flow import InstalledAppFlow, _WSGIRequestHandler, _RedirectWSGIApp +from google.auth.transport.requests import Request +import os +from pathlib import Path +import pickle +import re +import requests +import ssl +import webbrowser +import wsgiref.simple_server +import wsgiref.util +from urllib.parse import quote, urlparse, parse_qs +from .consts import AUTH_URL, CERT_FILE_PATH, KEY_FILE_PATH + +DROPBOX_FOLDER_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), 'exec_engine/credentials/dropbox') +TOKEN_FILE = os.path.join(DROPBOX_FOLDER_PATH, 'token.pickle') + +def main(): + creds = run_dropbox_flow() + # Save the credentials for the next run + if not os.path.isdir(DROPBOX_FOLDER_PATH): + os.mkdir(DROPBOX_FOLDER_PATH) + with open(TOKEN_FILE, 'wb') as token: + pickle.dump(creds, token) + +def run_dropbox_flow(): + host="localhost" + bind_addr=None + port=8000 + authorization_prompt_message="Please visit the link below to authorize: \n{url}" + success_message="authorization success, you may close the window" + open_browser=True, + redirect_uri_trailing_slash=True + timeout_seconds=None + browser=None + gorilla_user_scopes = 'user-read-private,user-read-email,playlist-modify-private,playlist-modify-public,user-top-read,user-read-recently-played' + + wsgi_app = _RedirectWSGIApp(success_message) + # Fail fast if the address is occupied + wsgiref.simple_server.WSGIServer.allow_reuse_address = False + local_server = wsgiref.simple_server.make_server( + bind_addr or host, port, wsgi_app, handler_class=_WSGIRequestHandler + ) + + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslctx.check_hostname = False # If set to True, only the hostname that matches the certificate will be accepted + sslctx.load_cert_chain(certfile=CERT_FILE_PATH, keyfile=KEY_FILE_PATH) + + local_server.socket = ssl.wrap_socket (local_server.socket, + keyfile=KEY_FILE_PATH, + certfile=CERT_FILE_PATH, server_side=True) + + redirect_uri_format = ( + "https://{}:{}/" if redirect_uri_trailing_slash else "https://{}:{}" + ) + redirect_uri = redirect_uri_format.format(host, local_server.server_port) + client_id = "8yht0rtna5qkk7v" + auth_url = f"https://www.dropbox.com/oauth2/authorize?client_id={client_id}&response_type=code&redirect_uri={quote(redirect_uri)}" + + if open_browser: + # if browser is None it defaults to default browser + webbrowser.get(browser).open(auth_url, new=1, autoraise=True) + + if authorization_prompt_message: + print(authorization_prompt_message.format(url=auth_url)) + + local_server.timeout = timeout_seconds + local_server.handle_request() + + # Note: using https here because oauthlib is very picky that + # OAuth 2.0 should only occur over https. + authorization_response = wsgi_app.last_request_uri.replace("http", "https") + code = parse_qs(urlparse(authorization_response).query)['code'][0] + local_server.server_close() + data = { + "service": "dropbox", + "code": code, + "redirect_uri": redirect_uri + } + r = requests.post(AUTH_URL, headers={}, json=data) + return r.json()['token'] + +main() \ No newline at end of file diff --git a/goex/authorizations/scripts/github_authorization.py b/goex/authorizations/scripts/github_authorization.py new file mode 100644 index 000000000..871a073ba --- /dev/null +++ b/goex/authorizations/scripts/github_authorization.py @@ -0,0 +1,83 @@ +from google_auth_oauthlib.flow import InstalledAppFlow, _WSGIRequestHandler, _RedirectWSGIApp +from google.auth.transport.requests import Request +import os +from pathlib import Path +import pickle +import re +import requests +import ssl +import webbrowser +import wsgiref.simple_server +import wsgiref.util +from urllib.parse import quote, urlparse, parse_qs +from .consts import AUTH_URL, CERT_FILE_PATH, KEY_FILE_PATH + +GITHUB_FOLDER_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), 'exec_engine/credentials/github') +TOKEN_FILE = os.path.join(GITHUB_FOLDER_PATH, 'token.pickle') + +def main(): + creds = run_github_flow() + # Save the credentials for the next run + if not os.path.isdir(GITHUB_FOLDER_PATH): + os.mkdir(GITHUB_FOLDER_PATH) + with open(TOKEN_FILE, 'wb') as token: + pickle.dump(creds, token) + +def run_github_flow(): + host="localhost" + bind_addr=None + port=8000 + authorization_prompt_message="Please visit the link below to authorize: \n{url}" + success_message="authorization success, you may close the window" + open_browser=True, + redirect_uri_trailing_slash=True + timeout_seconds=None + browser=None + gorilla_user_scopes = 'repo notifications user' + + wsgi_app = _RedirectWSGIApp(success_message) + # Fail fast if the address is occupied + wsgiref.simple_server.WSGIServer.allow_reuse_address = False + local_server = wsgiref.simple_server.make_server( + bind_addr or host, port, wsgi_app, handler_class=_WSGIRequestHandler + ) + + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslctx.check_hostname = False # If set to True, only the hostname that matches the certificate will be accepted + sslctx.load_cert_chain(certfile=CERT_FILE_PATH, keyfile=KEY_FILE_PATH) + + local_server.socket = ssl.wrap_socket (local_server.socket, + keyfile=KEY_FILE_PATH, + certfile=CERT_FILE_PATH, server_side=True) + + redirect_uri_format = ( + "https://{}:{}/" if redirect_uri_trailing_slash else "https://{}:{}" + ) + redirect_uri = redirect_uri_format.format(host, local_server.server_port) + client_id = "Iv1.6e751297f3ac0269" + auth_url = f"https://github.com/login/oauth/authorize?client_id={client_id}&redirect_uri={quote(redirect_uri)}&scopes={quote(gorilla_user_scopes)}" + + if open_browser: + # if browser is None it defaults to default browser + webbrowser.get(browser).open(auth_url, new=1, autoraise=True) + + if authorization_prompt_message: + print(authorization_prompt_message.format(url=auth_url)) + + local_server.timeout = timeout_seconds + local_server.handle_request() + + # Note: using https here because oauthlib is very picky that + # OAuth 2.0 should only occur over https. + authorization_response = wsgi_app.last_request_uri.replace("http", "https") + code = parse_qs(urlparse(authorization_response).query)['code'][0] + local_server.server_close() + data = { + "service": "github", + "code": code, + "redirect_uri": redirect_uri + } + r = requests.post(AUTH_URL, headers={}, json=data) + return r.json()['token'] + +main() diff --git a/goex/authorizations/scripts/gmail_authorization.py b/goex/authorizations/scripts/gmail_authorization.py new file mode 100644 index 000000000..289e94c3d --- /dev/null +++ b/goex/authorizations/scripts/gmail_authorization.py @@ -0,0 +1,98 @@ +from google_auth_oauthlib.flow import InstalledAppFlow, _WSGIRequestHandler, _RedirectWSGIApp +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +import os +from pathlib import Path +import pickle +import re +import requests +import ssl +import webbrowser +import wsgiref.simple_server +import wsgiref.util +from urllib.parse import quote, urlparse, parse_qs +from .consts import AUTH_URL, CERT_FILE_PATH, KEY_FILE_PATH + +# If modifying SCOPES, delete the token.pickle file +SCOPES = [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/calendar.readonly", + "openid", + "https://www.googleapis.com/auth/gmail.compose" , + "https://www.googleapis.com/auth/userinfo.profile", + "https://www.googleapis.com/auth/gmail.readonly", + ] + +GMAIL_FOLDER_PATH = os.path.join(os.path.dirname(Path(__file__).parent.parent), "exec_engine/credentials/gmail") +TOKEN_FILE = os.path.join(GMAIL_FOLDER_PATH, 'token.pickle') + +def main(): + creds = run_gmail_flow() + # Save the credentials for the next run + if not os.path.isdir(GMAIL_FOLDER_PATH): + os.mkdir(GMAIL_FOLDER_PATH) + with open(TOKEN_FILE, 'wb') as token: + pickle.dump(creds, token) + +def run_gmail_flow(): + host="localhost" + bind_addr=None + port=8000 + authorization_prompt_message= "Please visit the link below to authorize: \n{url}" + success_message="authorization success, you may close the window" + open_browser=True, + redirect_uri_trailing_slash=True + timeout_seconds=None + browser=None + + wsgi_app = _RedirectWSGIApp(success_message) + # Fail fast if the address is occupied + wsgiref.simple_server.WSGIServer.allow_reuse_address = False + local_server = wsgiref.simple_server.make_server( + bind_addr or host, port, wsgi_app, handler_class=_WSGIRequestHandler + ) + + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslctx.check_hostname = False # If set to True, only the hostname that matches the certificate will be accepted + sslctx.load_cert_chain(certfile=CERT_FILE_PATH, keyfile=KEY_FILE_PATH) + + local_server.socket = ssl.wrap_socket (local_server.socket, + keyfile=KEY_FILE_PATH, + certfile=CERT_FILE_PATH, server_side=True) + + redirect_uri_format = ( + "https://{}:{}/" if redirect_uri_trailing_slash else "https://{}:{}" + ) + redirect_uri = redirect_uri_format.format(host, local_server.server_port) + client_id = "486890796116-hfaa730qdb18ndhjs1aj195drjh5iple.apps.googleusercontent.com" + scopes_url = "+".join([quote(x) for x in SCOPES]) + auth_url =f"https://accounts.google.com/o/oauth2/auth?response_type=code&client_id={client_id}&redirect_uri={quote(redirect_uri)}&scope={scopes_url}&access_type=offline" + + if open_browser: + # if browser is None it defaults to default browser + webbrowser.get(browser).open(auth_url, new=1, autoraise=True) + + if authorization_prompt_message: + print(authorization_prompt_message.format(url=auth_url)) + local_server.timeout = timeout_seconds + local_server.handle_request() + # Note: using https here because oauthlib is very picky that + # OAuth 2.0 should only occur over https. + authorization_response = wsgi_app.last_request_uri.replace("http", "https") + code = parse_qs(urlparse(authorization_response).query)['code'][0] + local_server.server_close() + server_url = AUTH_URL + data = { + "service": "gmail-cli", + "code": code, + "redirect_uri": redirect_uri + } + r = requests.post(server_url, headers={}, json=data) + print(r.json()) + return Credentials.from_authorized_user_info(r.json()) + +main() + + + + diff --git a/goex/authorizations/scripts/slack_authorization.py b/goex/authorizations/scripts/slack_authorization.py new file mode 100644 index 000000000..37564f93b --- /dev/null +++ b/goex/authorizations/scripts/slack_authorization.py @@ -0,0 +1,89 @@ +from google_auth_oauthlib.flow import InstalledAppFlow, _WSGIRequestHandler, _RedirectWSGIApp +from google.auth.transport.requests import Request +import os +from pathlib import Path +import pickle +import re +import requests +import ssl +import webbrowser +import wsgiref.simple_server +import wsgiref.util +from .consts import AUTH_URL, CERT_FILE_PATH, KEY_FILE_PATH +from urllib.parse import quote, urlparse, parse_qs + +SLACK_FOLDER_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), 'exec_engine/credentials/slack') +TOKEN_FILE = os.path.join(SLACK_FOLDER_PATH, 'token.pickle') + +def main(): + creds = None + # Check if token.pickle file exists and is valid + if os.path.exists(TOKEN_FILE): + with open(TOKEN_FILE, 'rb') as token: + creds = pickle.load(token) + # If there are no (valid) credentials available, let the user log in + creds = run_slack_flow() + # Save the credentials for the next run + if not os.path.isdir(SLACK_FOLDER_PATH): + os.mkdir(SLACK_FOLDER_PATH) + with open(TOKEN_FILE, 'wb') as token: + pickle.dump(creds, token) + +def run_slack_flow(): + host="localhost" + bind_addr=None + port=8000 + authorization_prompt_message="Please visit the link below to authorize: \n{url}" + success_message="authorization success, you may close the window" + open_browser=True, + redirect_uri_trailing_slash=True + timeout_seconds=None + browser=None + gorilla_user_scopes = 'chat:write,im:read,im:history,users:read,users:read.email,groups:read,groups:write,groups:history,channels:read,channels:write,channels:history,mpim:read,mpim:write,mpim:history,reactions:read,reactions:write,files:write,files:read' + + wsgi_app = _RedirectWSGIApp(success_message) + # Fail fast if the address is occupied + wsgiref.simple_server.WSGIServer.allow_reuse_address = False + local_server = wsgiref.simple_server.make_server( + bind_addr or host, port, wsgi_app, handler_class=_WSGIRequestHandler + ) + + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslctx.check_hostname = False # If set to True, only the hostname that matches the certificate will be accepted + sslctx.load_cert_chain(certfile=CERT_FILE_PATH, keyfile=KEY_FILE_PATH) + + local_server.socket = ssl.wrap_socket (local_server.socket, + keyfile=KEY_FILE_PATH, + certfile=CERT_FILE_PATH, server_side=True) + + redirect_uri_format = ( + "https://{}:{}/" if redirect_uri_trailing_slash else "https://{}:{}" + ) + redirect_uri = redirect_uri_format.format(host, local_server.server_port) + client_id = "16753592806.6160184599591" + auth_url= f"https://slack.com/oauth/v2/authorize?response_type=code&client_id={client_id}&redirect_uri={quote(redirect_uri)}&access_type=offline&user_scope={gorilla_user_scopes}" + + if open_browser: + # if browser is None it defaults to default browser + webbrowser.get(browser).open(auth_url, new=1, autoraise=True) + + if authorization_prompt_message: + print(authorization_prompt_message.format(url=auth_url)) + + local_server.timeout = timeout_seconds + local_server.handle_request() + + # Note: using https here because oauthlib is very picky that + # OAuth 2.0 should only occur over https. + authorization_response = wsgi_app.last_request_uri.replace("http", "https") + code = parse_qs(urlparse(authorization_response).query)['code'][0] + local_server.server_close() + data = { + "service": "slack", + "code": code, + "redirect_uri": redirect_uri + } + r = requests.post(AUTH_URL, headers={}, json=data) + return r.json()['token'] + +main() diff --git a/goex/authorizations/scripts/spotify_authorization.py b/goex/authorizations/scripts/spotify_authorization.py new file mode 100644 index 000000000..b3affd997 --- /dev/null +++ b/goex/authorizations/scripts/spotify_authorization.py @@ -0,0 +1,89 @@ +from google_auth_oauthlib.flow import InstalledAppFlow, _WSGIRequestHandler, _RedirectWSGIApp +from google.auth.transport.requests import Request +import os +from pathlib import Path +import pickle +import re +import requests +import ssl +import webbrowser +import wsgiref.simple_server +import wsgiref.util +from urllib.parse import quote, urlparse, parse_qs +from .consts import AUTH_URL, CERT_FILE_PATH, KEY_FILE_PATH + +SPOTIFY_FOLDER_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), 'exec_engine/credentials/spotify') +TOKEN_FILE = os.path.join(SPOTIFY_FOLDER_PATH, 'token.pickle') + +def main(): + creds = None + # Check if token.pickle file exists and is valid + if os.path.exists(TOKEN_FILE): + with open(TOKEN_FILE, 'rb') as token: + creds = pickle.load(token) + # If there are no (valid) credentials available, let the user log in + creds = run_spotify_flow() + # Save the credentials for the next run + if not os.path.isdir(SPOTIFY_FOLDER_PATH): + os.mkdir(SPOTIFY_FOLDER_PATH) + with open(TOKEN_FILE, 'wb') as token: + pickle.dump(creds, token) + +def run_spotify_flow(): + host="localhost" + bind_addr=None + port=8000 + authorization_prompt_message="Please visit the link below to authorize: \n{url}" + success_message="authorization success, you may close the window" + open_browser=True, + redirect_uri_trailing_slash=True + timeout_seconds=None + browser=None + gorilla_user_scopes = 'user-read-private,user-read-email,playlist-modify-private,playlist-modify-public,user-top-read,user-read-recently-played' + + wsgi_app = _RedirectWSGIApp(success_message) + # Fail fast if the address is occupied + wsgiref.simple_server.WSGIServer.allow_reuse_address = False + local_server = wsgiref.simple_server.make_server( + bind_addr or host, port, wsgi_app, handler_class=_WSGIRequestHandler + ) + + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslctx.check_hostname = False # If set to True, only the hostname that matches the certificate will be accepted + sslctx.load_cert_chain(certfile=CERT_FILE_PATH, keyfile=KEY_FILE_PATH) + + local_server.socket = ssl.wrap_socket (local_server.socket, + keyfile=KEY_FILE_PATH, + certfile=CERT_FILE_PATH, server_side=True) + + redirect_uri_format = ( + "https://{}:{}/" if redirect_uri_trailing_slash else "https://{}:{}" + ) + redirect_uri = redirect_uri_format.format(host, local_server.server_port) + client_id = "d711dfc0d97440cb898b08fbdc2083c1" + auth_url = f"https://accounts.spotify.com/authorize?response_type=code&client_id={client_id}&redirect_uri={quote(redirect_uri)}&access_type=offline&scope={gorilla_user_scopes}" + + if open_browser: + # if browser is None it defaults to default browser + webbrowser.get(browser).open(auth_url, new=1, autoraise=True) + + if authorization_prompt_message: + print(authorization_prompt_message.format(url=auth_url)) + + local_server.timeout = timeout_seconds + local_server.handle_request() + + # Note: using https here because oauthlib is very picky that + # OAuth 2.0 should only occur over https. + authorization_response = wsgi_app.last_request_uri.replace("http", "https") + code = parse_qs(urlparse(authorization_response).query)['code'][0] + local_server.server_close() + data = { + "service": "spotify", + "code": code, + "redirect_uri": redirect_uri + } + r = requests.post(AUTH_URL, headers={}, json=data) + return r.json()['token'] + +main() \ No newline at end of file diff --git a/goex/cli.py b/goex/cli.py new file mode 100644 index 000000000..9a30545d8 --- /dev/null +++ b/goex/cli.py @@ -0,0 +1,379 @@ +import argparse +from simple_colors import * +from exec_engine.pipeline import * +from exec_engine.credentials.credentials_utils import * +from halo import Halo + +import os +from pathlib import Path +from authorizations.scripts.authorization_utils import authorize_service +from main import ExecutionEngine, PythonAPIExecutor + +from exec_engine.utils import SQL_Type, Filesystem_Type +from exec_engine.db_manager import MySQLManager, SQLiteManager +from dotenv import load_dotenv +import questionary + + +GORILLA_EMOJI = "🦍 " +SUCCESS = u'\u2713' +USER_CONFIG_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__))), "user_config.json") +default_config = {'max_attempt' : 1, + 'option': 2, + 'show_debug': True, + 'model': "gpt-4-turbo-preview", + 'undo': True, + 'dbtype': 'sqlite', + 'lfs_limit': 200, + 'fs_path': "" + } + +def insert_callback(service, key): + print(SUCCESS) + with Halo(text=f"{GORILLA_EMOJI}inserting creds...", spinner="dots"): + insert_creds(service, key, target = CREDS_FOLDER_PATH, cred_type="raw") + +def list_callback(): + print(list_creds(target = CREDS_FOLDER_PATH)) + +def restful_callback(prompt, generate_mode): + engine = ExecutionEngine(generate_mode=generate_mode) + # Specify a negation manager (e.g NaiveNegationAPIPairManager) when + # initializing PythonAPIExecutor to record the user feedback for the negated API + engine.api_executor = PythonAPIExecutor(engine.docker_sandbox) + creds, services = engine.api_executor.prepare_credentials(prompt) + + if not creds: + for service in list_supported_services(): + if service in prompt: + print('Warning: detect keyword {service} but {service} has not been authorized'.format(service=service)) + + if not services: + api_string_extension = "" + else: + api_string_extension = "from {services} API...".format(services=services) + with Halo(text=f"{GORILLA_EMOJI}fetching response {api_string_extension}".format(api_string_extension), spinner="dots"): + response, forward_call, backward_call = prompt_execute( + engine, prompt, services=services, creds=creds, max_attempt=get_config('max_attempt'), model=get_config('model')) + if response['output']: + print('\n', '\n'.join(response["output"][0])) + else: + print('\n', "execution failed with the following debug messages:") + print('\n', response['debug']) + return + + if default_config["undo"]: + answer = questionary.select("Are you sure you want to keep the changes", choices=["Commit", "Undo"]).ask() + + if answer.lower() == "undo": + # if there is a match with print_pattern, it's highly likely that it is only a print message + print_pattern = r'^\s*print\s*\((?:.|\n)*\)\s*$' + matches = re.search(print_pattern, backward_call, re.DOTALL) + if matches: + print(engine.api_executor.execute_api_call(backward_call, services)["output"]) + else: + print("Warning: the undo feature is still in beta mode and may cause irreversible changes\n" + + "Gorilla will execute the following code:\n{}".format(backward_call)) + confirmation = questionary.select("", choices=["Confirm Undo", "Cancel Undo"]).ask() + + if confirmation == "Confirm Undo": + print(engine.api_executor.execute_api_call(backward_call, services)["output"]) + else: + print("Abort undo, execution completed!") + + if engine.api_executor.negation_manager != None: + feedback = questionary.select("How would you rate the suggested negation API?", + choices=["Correct", "Incorrect", "Skip"]).ask() + + if feedback == "Correct": + engine.api_executor.negation_manager.insert_log(forward_call, backward_call, True) + elif feedback == "Incorrect": + engine.api_executor.negation_manager.insert_log(forward_call, backward_call, False) + print("Execution Completed!") + +def initialize_user_config(): + if os.path.exists(USER_CONFIG_PATH): + return + with open(USER_CONFIG_PATH, 'w') as j: + json.dump(default_config, j) + print("Config file created successfully.") + +def update_user_config(key, value): + with open(USER_CONFIG_PATH, 'r') as j: + oldconfig = json.load(j) + if key == 'max_attempt' or key == 'option' or key == 'lfs_limit': + value = int(value) + elif key == 'show_debug': + value = value.lower() == 'true' + elif key == 'fs_path': + value = os.path.join(os.getcwd(), value) + if not os.path.exists(value) and not os.path.isdir(value): + print("Please make sure you enter a valid directory path!") + return + modified = False + if oldconfig[key] != value: + modified = True + oldconfig[key] = value + if modified: + with open(USER_CONFIG_PATH, 'w') as j: + json.dump(oldconfig, j) + print("Config file modified successfully.") + +def get_config(key): + with open(USER_CONFIG_PATH, 'r') as j: + config = json.load(j) + return config[key] + +def authorize_callback(services): + supported_services = list_supported_services() + for service in services: + if service in supported_services: + try: + authorize_service(service) + except Exception as e: + print(e) + print("Failed to authorize user's {service} account".format(service=service)) + else: + print("{service} is currently not supported".format(service=service)) + +def fs_callback(prompt, generate_mode): + path = get_config('fs_path') + if not path: + path = os.getcwd() + path = os.path.abspath(path) + + engine = ExecutionEngine(path=path, generate_mode=generate_mode) + option = get_config('option') + engine.initialize_fs(debug_path=path, git_init=option == 2) + + if option == 1: + engine.set_dry_run(Filesystem_Type, True) + else: + engine.set_dry_run(Filesystem_Type, False) + + api_call, neg_api_call = engine.gen_api_pair(prompt, Filesystem_Type, None, model=get_config('model')) + print(black("Do you want to execute the following filesystem command...", 'bold') + '\n' + magenta(api_call, 'bold')) + answer = questionary.select("", + choices=["Yes", "No"]).ask() + if answer == "No": + print("Execution abandoned.") + return + + try: + engine.exec_api_call(api_call=api_call, api_type=Filesystem_Type, debug_neg=neg_api_call) + except RuntimeError as e : + print(f"Execution Failed: {e}") + return + + option_to_method = { + 1: 'negation call', + 2: 'git reset' + } + answer = questionary.select("Are you sure you want to keep the changes", + choices=["Commit", "Undo" + " ({})". + format(option_to_method[option])]).ask() + + if option == 2: + if answer == "Commit": + commit_msg = questionary.text("Enter a commit message [Optional]: ").ask() + engine.commit_api_call(Filesystem_Type, commit_msg) + print("Execution commited.") + else: + engine.undo_api_call(Filesystem_Type, option=option) + print("Execution undone.") + else: + if answer == "Commit": + print("Execution completed.") + else: + engine.exec_api_call(neg_api_call, api_type=Filesystem_Type) + print("Execution undone.") + +def remove_creds_callback(services): + try: + remove_creds(services) + except Exception as e: + print(e) + print("An unexpected error occured while removing credentials") + +def db_callback(prompt, generate_mode): + config = { + 'user': os.environ.get('DATABASE_USER'), + 'password': os.environ.get('DATABASE_PASSWORD'), + 'host': os.environ.get('DATABASE_HOST'), + 'database': os.environ.get('DATABASE_NAME'), + 'path': os.environ.get('DATABASE_PATH') + } + + engine = ExecutionEngine(generate_mode=generate_mode) + + db_type = get_config('dbtype') + db_manager = None + try: + if db_type == 'mysql': + db_manager = MySQLManager(config, docker_sandbox=engine.docker_sandbox) + elif db_type == 'sqlite': + db_manager = SQLiteManager(config, docker_sandbox=engine.docker_sandbox) + except Exception as e: + print(f"Error during {db_type} Manager Init: {e}") + return + db_manager.connect() + + option = get_config('option') + if option == 1: + engine.set_dry_run(SQL_Type, True) + else: + engine.set_dry_run(SQL_Type, False) + + engine.initialize_db(debug_manager=db_manager) + api_call, neg_api_call = engine.gen_api_pair(prompt, SQL_Type, None, model=get_config('model')) + if neg_api_call == None and option == 1: + print("Error: option 1 requires negation API call. neg_api_call is None.") + return + print(black("Do you want to execute the following database command...", 'bold') + '\n' + magenta(api_call, 'bold')) + answer = questionary.select("", + choices=["Yes", "No"]).ask() + if answer == "No": + print("Execution abandoned.") + return + + try: + engine.exec_api_call(api_call=api_call, api_type=SQL_Type, debug_neg=neg_api_call) + if option == 1: + engine.commit_api_call(SQL_Type) + except RuntimeError as e : + print(f"Execution Failed: {e}") + return + + + option_to_method = { + 1: 'negation call', + 2: 'db rollback' + } + answer = questionary.select("Are you sure you want to keep the changes", + choices=["Commit", "Undo" + " ({})". + format(option_to_method[option])]).ask() + + if option == 2: + if answer == "Commit": + engine.commit_api_call(SQL_Type) + print("Execution commited.") + else: + engine.undo_api_call(SQL_Type, option=option) + print("Execution undone.") + else: + if answer == "Commit": + print("Execution completed!") + else: + engine.exec_api_call(neg_api_call, api_type=SQL_Type) + print("Execution undone.") + + +def exit_with_help_message(parser): + print(green("To execute a prompt with a specified execution type", ['bold'])) + # retrieve subparsers from parser + subparsers_actions = [ + action for action in parser._actions + if isinstance(action, argparse._SubParsersAction)] + # there will probably only be one subparser_action, + # but better save than sorry + for subparsers_action in subparsers_actions: + # get all subparsers and print help + for choice, subparser in subparsers_action.choices.items(): + print(subparser.format_help()) + + print(green("To perform other Gorilla-x operations", ['bold'])) + parser.print_help() + + parser.exit() + +class _HelpAction(argparse._HelpAction): + + def __call__(self, parser, namespace, values, option_string=None): + exit_with_help_message(parser) + +class ArgumentParser(argparse.ArgumentParser): + + def error(self, message): + exit_with_help_message(self) + + +def main(): + initialize_user_config() + parser = ArgumentParser(add_help=False) + subparser = parser.add_subparsers(dest='command') + + execute_parser = subparser.add_parser("execute", add_help=False) + execute_parser.add_argument("-prompt", nargs='*', metavar='prompt', help="prompt for Gorilla-x to execute") + execute_parser.add_argument("-type", nargs=1, metavar='type', help="specify the execution type as either 'rest', 'db', or 'fs'") + execute_parser.add_argument("-generate_mode", metavar='gen_mode', help="specify how to use the LLM, either 'default', 'function_in_context' or 'function_calling_native'", default='default') + + parser.add_argument('--help', action=_HelpAction) + parser.add_argument('-insert_creds', action='store', metavar=('service', 'key'), nargs=2, help="Inserts the service-key pair to Gorilla's secret store.") + parser.add_argument('-list_creds', action='store_true', help="Lists all the currently registered credentials.") + parser.add_argument('-authorize', action='store', metavar='service', nargs=1, help="Perform OAuth2 authorization and retrieve access token from the service") + parser.add_argument('-remove_creds', action='extend', metavar='service', nargs="+", help="Removes previously authorized credentials. Enter ALL as parameter to delete all creds") + parser.add_argument('-set_config', action='store', metavar=('name', 'value'), nargs=2, help="Updates the user config with the corresponding key value pairs") + + try: + args = parser.parse_args() + except argparse.ArgumentError: + parser.print_help() + return + + # load the environment variables + load_dotenv() + + if args.command == "execute": + if args.prompt and args.type: + prompt = " ".join(args.prompt) + apitype = args.type[0].lower() + if "rest" in apitype: + restful_callback(prompt, args.generate_mode) + elif "db" in apitype: + db_callback(prompt, args.generate_mode) + elif "fs" in apitype: + fs_callback(prompt, args.generate_mode) + else: + print("Error: invalid execution type. The execution types Gorilla-x currently support are: \n" + + " 1. RESTful (rest)\n" + + " 2. Database (db)\n" + + " 3. Filesystem (fs)") + return + else: + print("execute requires -prompt and -type to be provided") + return + + if args.authorize: + authorize_callback(args.authorize) + elif args.remove_creds: + remove_creds_callback(args.remove_creds) + elif args.list_creds: + list_callback() + elif args.insert_creds: + key = args.insert_creds[0] + path = args.insert_creds[1] + insert_callback(key, path) + elif args.set_config: + key = args.set_config[0] + value = args.set_config[1] + if key.lower() == 'max_attempt': + if not value.isnumeric(): + print("Please enter a positive integer.") + return + else: + value = int(value) + key = 'max_attempt' + elif key.lower() == 'model': + if value.isdigit(): + print("Please enter a valid model version.") + else: + value = value.lower() + key = "model" + + update_user_config(key, value) + else: + exit_with_help_message(parser) + + +if __name__ == "__main__": + main() diff --git a/goex/demo/env_setup.py b/goex/demo/env_setup.py new file mode 100644 index 000000000..797e5d82d --- /dev/null +++ b/goex/demo/env_setup.py @@ -0,0 +1,51 @@ +""" +Setup Script for getting .env populated using .env.example +""" +import os +from pathlib import Path + +ROOT_FOLDER_PATH = os.path.dirname(Path(os.path.realpath(__file__)).parent) + +def parse_env_example(file_path): + """ + Parses the .env.example file and returns a dictionary of the variables and their default values. + """ + variables = {} + with open(file_path, 'r') as file: + for line in file: + if line.strip() and not line.startswith('#'): + key, value = line.strip().split('=', 1) + variables[key] = value + return variables + +def create_filled_env(file_path, variables): + """ + Creates a new .env file based on the variables dictionary provided by the user. + """ + with open(file_path, 'w') as file: + for key, value in variables.items(): + file.write(f'{key}={value}\n') + +def main(): + env_path = '.env' + if os.path.exists(env_path): + env_example_path = env_path + else: + env_example_path = '.env.example' + + variables = parse_env_example(env_example_path) + + print("Please fill out the following fields for the .env config [Leave empty for default]:") + + for key in variables.keys(): + user_input = input(f"{key} [{variables[key]}]: ").strip() + if 'PATH' in key: + user_input = os.path.abspath(user_input) if user_input else os.path.abspath(variables[key]) + variables[key] = user_input if user_input else variables[key] + + create_filled_env(env_path, variables) + + print(f"Your configuration has been saved to {env_path}.") + +if __name__ == "__main__": + main() diff --git a/goex/demo/mysql_setup.py b/goex/demo/mysql_setup.py new file mode 100644 index 000000000..817494b0b --- /dev/null +++ b/goex/demo/mysql_setup.py @@ -0,0 +1,45 @@ +""" +Setup script for MySQL +""" + +import subprocess +import os +import sys +from pathlib import Path +from dotenv import load_dotenv + +load_dotenv() + +if __name__ == "__main__": + # Check if the database name was provided as a command-line argument + if len(sys.argv) < 2: + print("Usage: python3 mysql_setup.py database_name") + sys.exit(1) + + ROOT_FOLDER_PATH = os.path.dirname(Path(os.path.realpath(__file__)).parent) + + mysql_username = os.environ.get('DATABASE_USER') + mysql_password = os.environ.get('DATABASE_PASSWORD') + database_name = sys.argv[1] + dump_file_path = os.path.join(ROOT_FOLDER_PATH, "docker/mysql_docker/checkpoints/database_dump.sql") + + # Command to create the database if it doesn't exist + create_db_command = f'mysql -u{mysql_username} -p{mysql_password} -e "CREATE DATABASE IF NOT EXISTS {database_name};"' + + # Command to import the .sql file into the MySQL database + import_command = f'mysql -u{mysql_username} -p{mysql_password} {database_name} < {dump_file_path}' + + # Execute the create database command + try: + subprocess.run(create_db_command, shell=True, check=True) + print(f"Database '{database_name}' created (if it didn't already exist).") + except subprocess.CalledProcessError as e: + print(f"Error creating database: {e}") + sys.exit(1) + + # Execute the import command + try: + subprocess.run(import_command, shell=True, check=True) + print("Import successful") + except subprocess.CalledProcessError as e: + print(f"Error during import: {e}") diff --git a/goex/docker/docker/dockerfile b/goex/docker/docker/dockerfile new file mode 100644 index 000000000..d6fecc9e7 --- /dev/null +++ b/goex/docker/docker/dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:latest + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y python3-pip python3-dev \ + && cd /usr/local/bin \ + && ln -s /usr/bin/python3 python \ + && pip3 --no-cache-dir install --upgrade pip \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /sandbox + +COPY requirements.txt /sandbox +RUN pip3 install -r requirements.txt + +COPY python_executor.py /sandbox diff --git a/goex/docker/docker/python_executor.py b/goex/docker/docker/python_executor.py new file mode 100644 index 000000000..eee20b5d7 --- /dev/null +++ b/goex/docker/docker/python_executor.py @@ -0,0 +1,17 @@ +from io import StringIO +from contextlib import redirect_stdout +import sys +import os + +def code_execute(): + f = StringIO() + code = os.environ['CODE'] + with redirect_stdout(f): + exec(code, globals()) + + exec_stdout = f.getvalue() + print(exec_stdout) + return exec_stdout + +if __name__ == "__main__": + globals()[sys.argv[1]]() diff --git a/goex/docker/misc/images.json b/goex/docker/misc/images.json new file mode 100644 index 000000000..ae6bfb47f --- /dev/null +++ b/goex/docker/misc/images.json @@ -0,0 +1 @@ +{ "image_hash": "docker_image_sha256" } diff --git a/goex/docker/mysql_docker/checkpoints/database_dump.sql b/goex/docker/mysql_docker/checkpoints/database_dump.sql new file mode 100644 index 000000000..90fa37f86 --- /dev/null +++ b/goex/docker/mysql_docker/checkpoints/database_dump.sql @@ -0,0 +1,53 @@ +-- MySQL dump 10.13 Distrib 8.3.0, for macos14 (arm64) +-- +-- Host: localhost Database: tester +-- ------------------------------------------------------ +-- Server version 8.3.0 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!50503 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `students` +-- + +DROP TABLE IF EXISTS `students`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `students` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(100) NOT NULL, + `year` int NOT NULL, + `major` varchar(100) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `students` +-- + +LOCK TABLES `students` WRITE; +/*!40000 ALTER TABLE `students` DISABLE KEYS */; +INSERT INTO `students` VALUES (1,'Jane Doe',2,'Computer Science'),(2,'ray Doe',2,'Computer Science'); +/*!40000 ALTER TABLE `students` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2024-02-28 15:54:24 diff --git a/goex/docker/mysql_docker/container_setup/docker-entrypoint.sh b/goex/docker/mysql_docker/container_setup/docker-entrypoint.sh new file mode 100644 index 000000000..14164e9b0 --- /dev/null +++ b/goex/docker/mysql_docker/container_setup/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# docker-entrypoint.sh + +# Start MySQL service +service mysql start + +# Wait for MySQL to start up +sleep 10 + +exec "$@" diff --git a/goex/docker/mysql_docker/container_setup/mysql.sh b/goex/docker/mysql_docker/container_setup/mysql.sh new file mode 100644 index 000000000..224f439ce --- /dev/null +++ b/goex/docker/mysql_docker/container_setup/mysql.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Install MySQL server +apt-get update +apt install -y mysql-server + +# Start MySQL service +usermod -d /var/lib/mysql/ mysql +service mysql start + +# Configure MySQL +mysql -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY '';" +mysql -e "FLUSH PRIVILEGES" diff --git a/goex/docker/mysql_docker/dockerfile b/goex/docker/mysql_docker/dockerfile new file mode 100644 index 000000000..50613bbaa --- /dev/null +++ b/goex/docker/mysql_docker/dockerfile @@ -0,0 +1,30 @@ +FROM ubuntu:latest + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y python3-pip python3-dev \ + && cd /usr/local/bin \ + && ln -s /usr/bin/python3 python \ + && pip3 --no-cache-dir install --upgrade pip \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /sandbox + +COPY requirements.txt /sandbox +RUN pip3 install -r requirements.txt + +COPY container_setup/mysql.sh /sandbox +COPY container_setup/docker-entrypoint.sh /usr/local/bin + +RUN bash mysql.sh + +# Make the entrypoint script executable +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +# Use your entrypoint script +ENTRYPOINT ["docker-entrypoint.sh"] + +COPY python_executor.py /sandbox + +COPY checkpoints/database_dump.sql /sandbox diff --git a/goex/docker/mysql_docker/python_executor.py b/goex/docker/mysql_docker/python_executor.py new file mode 100644 index 000000000..eee20b5d7 --- /dev/null +++ b/goex/docker/mysql_docker/python_executor.py @@ -0,0 +1,17 @@ +from io import StringIO +from contextlib import redirect_stdout +import sys +import os + +def code_execute(): + f = StringIO() + code = os.environ['CODE'] + with redirect_stdout(f): + exec(code, globals()) + + exec_stdout = f.getvalue() + print(exec_stdout) + return exec_stdout + +if __name__ == "__main__": + globals()[sys.argv[1]]() diff --git a/goex/docker/sqllite_docker/dockerfile b/goex/docker/sqllite_docker/dockerfile new file mode 100644 index 000000000..6e13b92f6 --- /dev/null +++ b/goex/docker/sqllite_docker/dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:latest + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y python3-pip python3-dev \ + && cd /usr/local/bin \ + && ln -s /usr/bin/python3 python \ + && pip3 --no-cache-dir install --upgrade pip \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /sandbox + +COPY requirements.txt /sandbox +RUN pip3 install -r requirements.txt + +COPY python_executor.py /sandbox + +COPY example_sqlite.db /sandbox + diff --git a/goex/docker/sqllite_docker/example_sqlite.db b/goex/docker/sqllite_docker/example_sqlite.db new file mode 100644 index 000000000..8cbcc7a78 Binary files /dev/null and b/goex/docker/sqllite_docker/example_sqlite.db differ diff --git a/goex/docker/sqllite_docker/python_executor.py b/goex/docker/sqllite_docker/python_executor.py new file mode 100644 index 000000000..eee20b5d7 --- /dev/null +++ b/goex/docker/sqllite_docker/python_executor.py @@ -0,0 +1,17 @@ +from io import StringIO +from contextlib import redirect_stdout +import sys +import os + +def code_execute(): + f = StringIO() + code = os.environ['CODE'] + with redirect_stdout(f): + exec(code, globals()) + + exec_stdout = f.getvalue() + print(exec_stdout) + return exec_stdout + +if __name__ == "__main__": + globals()[sys.argv[1]]() diff --git a/goex/examples.py b/goex/examples.py new file mode 100644 index 000000000..4888dfd71 --- /dev/null +++ b/goex/examples.py @@ -0,0 +1,329 @@ +"This file lists some popular examples on how to use Gorilla Execution Engine" + + + +""" +PythonAPIExecutor + - Create a python command in attempt to perform actions for the given prompt + - If key(s) are needed, make sure they are in secret_store.json +""" +import os +from exec_engine.db_manager import MySQLManager, SQLiteManager +from main import ExecutionEngine, PythonAPIExecutor +from exec_engine.utils import SQL_Type, RESTful_Type, Filesystem_Type +from pathlib import Path + +from dotenv import load_dotenv + +ROOT_FOLDER_PATH = os.path.dirname(Path(os.path.realpath(__file__))) + +def mysql_insert_new_row_with_dry_run(api_call=None): + load_dotenv() + # DB tests + mysql_config = { + 'user': os.environ.get('DATABASE_USER'), + 'password': os.environ.get('DATABASE_PASSWORD'), + 'host': os.environ.get('DATABASE_HOST'), + 'database': os.environ.get('DATABASE_NAME') + } + if not api_call or not neg_api_call: + api_call = "INSERT INTO students (name, year, major) VALUES ('Roy Huang', 4, 'Computer Science');" + neg_api_call = """DELETE FROM students WHERE id IN ( + SELECT * FROM ( + SELECT MAX(id) FROM students + ) AS subquery + ); + """ + check_call = "SELECT * FROM students;" + + engine = ExecutionEngine() + engine.set_dry_run(SQL_Type, True) + + db_manager = MySQLManager(mysql_config, docker_sandbox=engine.docker_sandbox) + db_manager.connect() + engine.initialize_db(debug_manager=db_manager) + + # avoid dry running the SELECTs + print("DB before the insertion") + print(engine._exec_sql_call(check_call)) + engine.exec_api_call(api_call=api_call, debug_neg=neg_api_call, api_type=SQL_Type) + print("DB after the insertion") + print(engine._exec_sql_call(check_call)) + engine._exec_sql_call(neg_api_call) + print("DB after the reversion") + print(engine._exec_sql_call(check_call)) + +def create_new_file(): + test_dir = 'test' + os.makedirs(test_dir, exist_ok=True) + engine = ExecutionEngine() + engine.initialize_fs(debug_path=test_dir) + # Example usage + engine.exec_api_call('ls -a', Filesystem_Type) + engine.exec_api_call('echo "Hello, World!" > hello.txt', Filesystem_Type) + engine.exec_api_call('ls -a', Filesystem_Type) + engine.exec_api_call('cat hello.txt', Filesystem_Type) + +def prompt_api_execute(prompt): + engine = ExecutionEngine() + engine.api_executor = PythonAPIExecutor(engine.docker_sandbox) + creds, services = engine.api_executor.prepare_credentials(prompt) + forward_call, backward_call = engine.gen_api_pair(prompt, api_type=RESTful_Type, credentials=creds, model="gpt-4-turbo-preview") + print(forward_call) + output = engine.api_executor.execute_api_call(forward_call, services) + return output + +def send_slack_message(content, display_name): + prompt = """ + Send the message ({content}) to @{display_name} on slack . + """.format(content=content, display_name=display_name.replace(" ", "")) + + print(prompt_api_execute(prompt)) + +def delete_slack_message(display_name): + prompt = """ + Delete the latest message I sent to the user {display_name} on slack direct message. + """.format(display_name=display_name.replace(" ", "")).lower() + + print(prompt_api_execute(prompt)) + +def latest_n_emails_gmail(n): + prompt = """ + Who are the senders of the {n} most recent email in my gmail inbox? + """.format(n=n) + + print(prompt_api_execute(prompt)) + +def ask_general_question(question): + print(prompt_api_execute(question)) + +""" +FS management system +""" +def full_file_system_demo(): + test_dir = 'test' + os.makedirs(test_dir, exist_ok=True) + engine = ExecutionEngine(path=test_dir) + engine.initialize_fs(debug_path=test_dir) + # Example usage + engine._exec_filesystem_call('ls -a') + engine.exec_api_call('echo "Hello, World!" > hello.txt', Filesystem_Type) + engine._exec_filesystem_call('ls -a') + engine._exec_filesystem_call('cat hello.txt') + + engine.commit_api_call(Filesystem_Type) + print('\n\nCommited!\n\n') + + engine.exec_api_call('echo "Bad File!" > not_good.txt', Filesystem_Type) + engine._exec_filesystem_call('ls -a', Filesystem_Type) + engine._exec_filesystem_call('cat not_good.txt', Filesystem_Type) + + + engine.undo_api_call(Filesystem_Type) + print('\n\nReverted!\n\n') + + engine.exec_api_call('ls -a', Filesystem_Type) + + +def mysql_insert_new_row_no_dry_run(api_call=None): + load_dotenv() + # DB tests + mysql_config = { + 'user': os.environ.get('DATABASE_USER'), + 'password': os.environ.get('DATABASE_PASSWORD'), + 'host': os.environ.get('DATABASE_HOST'), + 'database': os.environ.get('DATABASE_NAME') + } + if not api_call or not neg_api_call: + api_call = "INSERT INTO students (name, year, major) VALUES ('Roy Huang', 4, 'Computer Science');" + neg_api_call = """ + DELETE FROM students WHERE id IN ( + SELECT * FROM ( + SELECT MAX(id) FROM students + ) AS subquery + ); + """ + check_call = "SELECT * FROM students;" + + engine = ExecutionEngine() + db_manager = MySQLManager(mysql_config, docker_sandbox=engine.docker_sandbox) + db_manager.connect() + engine.initialize_db(debug_manager=db_manager) + + print('Current State:') + print(engine._exec_sql_call(check_call)) + + engine.exec_api_call("INSERT INTO students (name, year, major) VALUES ('Ray Huang', 4, 'Computer Science');", api_type=SQL_Type) + + print('New Commited State:') + print(engine._exec_sql_call(check_call)) + + engine.commit_api_call(SQL_Type) + + engine.exec_api_call("INSERT INTO students (name, year, major) VALUES ('Wrong dude', 1, 'high schooler');", api_type=SQL_Type) + + print('Uncommited Changed State:') + print(engine._exec_sql_call(check_call)) + + engine.undo_api_call(SQL_Type) + + print('Previous Commited Changed State:') + print(engine._exec_sql_call(check_call)) + +def fs_all_in(): + test_dir = 'test' + os.makedirs(test_dir, exist_ok=True) + engine = ExecutionEngine() + engine.initialize_fs(debug_path=test_dir) + # Example usage + engine.exec_api_call('ls -a') + engine.exec_api_call('echo "Hello, World!" > hello.txt') + engine.exec_api_call('ls -a') + engine.exec_api_call('cat hello.txt') + + engine.commit_api_call(Filesystem_Type) + print('\n\nCommited!\n\n') + + engine.exec_api_call('echo "Bad File!" > not_good.txt') + engine.exec_api_call('ls -a') + engine.exec_api_call('cat not_good.txt') + + engine.undo_api_call(SQL_Type) + print('\n\nReverted!\n\n') + + engine.exec_api_call('ls -a') + +def mysql_end_to_end_insert(): + load_dotenv() + # DB tests + mysql_config = { + 'user': os.environ.get('DATABASE_USER'), + 'password': os.environ.get('DATABASE_PASSWORD'), + 'host': os.environ.get('DATABASE_HOST'), + 'database': os.environ.get('DATABASE_NAME') + } + + check_call = "SELECT * FROM students;" + + engine = ExecutionEngine() + engine.set_dry_run(SQL_Type, True) + + db_manager = MySQLManager(mysql_config, docker_sandbox=engine.docker_sandbox) + db_manager.connect() + engine.initialize_db(debug_manager=db_manager) + + prompt = "i want to insert a new student name Shishir Patil who's a 1st year and a computer science major into the students table" + # prompt = "i want to delete a student named ray Doe in the students table" + + print("Before execution:") + print(engine._exec_sql_call(check_call)) + + engine.run_prompt(prompt, SQL_Type) + + print("After execution:") + print(engine._exec_sql_call(check_call)) + +def sqlite_insert_with_dry_run_llm_reversion(): + engine = ExecutionEngine() + + db_path = os.path.join(ROOT_FOLDER_PATH, 'docker/sqllite_docker/example_sqlite.db') + config = {'path': db_path} + db_manager = SQLiteManager(config, engine.docker_sandbox) + db_manager.connect() + + engine.initialize_db(debug_manager=db_manager) + engine.set_dry_run(SQL_Type, True) + check_call = "SELECT * FROM projects;" + + prompt = "i want to insert a new example row into the projects table" + + print("Before execution:") + print(engine._exec_sql_call(check_call)) + + engine.run_prompt(prompt, SQL_Type) + + print("After execution:") + print(engine._exec_sql_call(check_call)) + + engine.undo_api_call(SQL_Type) + +def fs_joke_prompt_demo(): + test_dir = 'test' + os.makedirs(test_dir, exist_ok=True) + engine = ExecutionEngine(path=test_dir) + engine.initialize_fs(debug_path=test_dir) + engine.set_dry_run(Filesystem_Type, True) + # Example usage + + print("Before execution:") + print(engine._exec_filesystem_call('ls -a')) + + engine.run_prompt("I want to create a file named joke.txt with a witty joke inside", Filesystem_Type) + engine.commit_api_call(Filesystem_Type) + + print("After execution:") + print(engine._exec_filesystem_call('ls -a')) + print(engine._exec_filesystem_call('cat joke.txt')) + + + +if __name__ == "__main__": + + """ + NOTE: Feel free to uncomment any of the tests below to run the demo + + IMPORTANT: Follow the README to get set up + """ + + """ + File System Examples + + You can see the actual file changes inside test/ directory + """ + # full_file_system_demo() + # create_new_file() + # fs_joke_prompt_demo() + + """ + MySQL Examples + """ + # mysql_insert_new_row_with_dry_run() + # mysql_insert_new_row_no_dry_run() + # mysql_end_to_end_insert() + + """ + sqllite Examples + """ + # sqlite_insert_with_dry_run_llm_reversion() + + """ + RESTful Examples + """ + # SLACK (requires OAuth) + # send_slack_message("yo", "Shishir Patil") + # delete_slack_message("Shishir Patil") + + # SPOTIFY (requires OAuth) + # spotify is sometimes a bit spotty (heh) and does not always work + # create a new file in my dropbox account with some jokes + # ask_general_question("find the user info associated with my spotify account") + + # DROPBOX (requires OAuth) + # ask_general_question("create a new file in my dropbox account with some jokes") + # ask_general_question("list all files in my dropbox account") + + # STRIPE (requires API key) + # ask_general_question("create a new customer with email aaronhao@berkeley.edu on stripe") + # ask_general_question("add 300 dollars to the balance of the customer with email aaronhao@berkeley.edu on stripe") + + # GITHUB (requires oauth) + # ask_general_question("get my profile information from github") + # ask_general_question("raise a github issue for my repository at [REPO] with text: [TEXT]") + + # DISCORD (requires oauth) + # ask_general_question("what is the email associated with my discord account") + + # ask_general_question("What's the top rising stock today on alphavantage?") + # ask_general_question("What's the weather today in San Francisco?") + # ask_general_question("What's the estimated time of arrival to the Salesforce tower Sanfrancisco if I leave berkeley right now with Bart") + pass diff --git a/goex/exec_engine/README.md b/goex/exec_engine/README.md new file mode 100644 index 000000000..284f4e6e8 --- /dev/null +++ b/goex/exec_engine/README.md @@ -0,0 +1,26 @@ +# Gorilla-X + +## Execution Workflows + +We categorize API calls into several categories, namely, RESTful API, filesystem APIs, database APIs, and POSIX APIs. A main concern we are trying to address is how to safely handle situations where LLMs may produce code that leads to unintended behaviors. + +### File System + +With filesystems API calls, they will be handled by an wrapper class `FSManager` that handles all the code execution as well as reversion functionalities. Upon initialization inside `ExecutionEngine`, the newly created FSManager based on either a user specified directory or the CWD will be initialized to track changes. + +- Initialize a Git repo to keep track of changes + - if it is a directory of size > 200MB, we use Git LFS + +`FSManager` will handle all of the history reversion based on standard Git principles to provide a robust version control system for easy rollbacks to the pre-LLM execution state. + +### Database + +With Database API calls, they will be handled by a wrapper interface `DBManager` that gets implemented into specific managers for each DB type (i.e. `MySQLManager`). + +The `DBManager` will handle execution of the API call, as well as keeping track of the current open transaction awaiting commit or rollback if the user decides to not perform dry run testing of the API calls (option 1). + +If the user has the dry run testing option on (option 2), then before the API call gets executed locally, a docker container will be spawned with a copy of the current database you are working with (`exec_engine/utils.py`). + +- the setup required for each DB will be performed in a shell script located in `docker/docker/container_setup` + +Afterwards, a Python script will run with the API call wrapped around a reversion tester to see if the state before running the API call and after the API call + its negation are the same. This gets captured and sent back as the dry run result. From their, the CLI or GUI can prompt the user to confirm or cancel the operation if it is deemed to be irreversible by the generated negation API. diff --git a/goex/exec_engine/__init__.py b/goex/exec_engine/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/goex/exec_engine/api_executor.py b/goex/exec_engine/api_executor.py new file mode 100644 index 000000000..398730b81 --- /dev/null +++ b/goex/exec_engine/api_executor.py @@ -0,0 +1,105 @@ +"""This module will execute API calls and their reversals.""" + +import subprocess +import json +from exec_engine.db_manager import DBManager + +from exec_engine.credentials.credentials_utils import creds_from_prompt, CREDS_FOLDER_PATH +from exec_engine.docker_sandbox import * + +from exec_engine.utils import RESTful_Type +from exec_engine.pipeline import generate_reverse_command +from exec_engine.negation_manager import NegationAPIPairManager, NaiveNegationAPIPairManager + +from pathlib import Path + +CUR_DIR = os.path.dirname(Path(os.path.realpath(__file__))) + +def code_add_dummy_argument(python_code): + dummy_args = json.load(open('./function/dummy_key.json', 'r')) + for dummy_args_key in dummy_args: + if isinstance(dummy_args[dummy_args_key], str): + dummy_args[dummy_args_key] = "\"" + dummy_args[dummy_args_key] + "\"" + python_code = python_code.replace("\"<<"+dummy_args_key+"_placeholder>>\"", dummy_args[dummy_args_key]) + return python_code + +class APIExecutor: + """Base Class for all API executors + + Should be stateless and should not have any attributes. + + Attributes: + None + + Methods: + execute_api_call: Execute API call + """ + def __init__(self): + return None + + def execute_api_call(self, command: str) -> int: + """Execute API call. + + Args: + command (str): API call to execute. + """ + raise NotImplementedError + + def set_execution_environment(self, env, docker_sandbox: DockerSandbox = None): + if env == "local": + self.env = "local" + return + elif env == "docker": + self.env = "docker" + self.docker_client = "docker_client" + return + else: + print('env can only be set to "docker" or "local"') + +class PythonAPIExecutor(APIExecutor): + """Executes Python API calls + + Methods: + execute_api_call: Execute API call + """ + def __init__(self, docker_sandbox: DockerSandbox = None, negation_manager: NaiveNegationAPIPairManager = None, path: str = CUR_DIR): + self.env = None + self.docker_sandbox = docker_sandbox + if negation_manager != None: + self.negation_manager = negation_manager(path) + else: + self.negation_manager = None + + def prepare_credentials(self, prompt:str, technique="lut"): + credentials = creds_from_prompt(prompt, CREDS_FOLDER_PATH, technique) + try: + services = [service_name for service_name, value, file_type in credentials] + except: + raise Exception("Error: credentials have to be passed in as a list of [service_name, value, cred_type] pairs") + return credentials, services + + # try_get_backward_call will try to see if there is a corresponding backward call for the input forward call + # if not, it will prompt the LLM to come up with one + def try_get_backward_call(self, forward_call, prompt, credentials, api_type, generate_mode='default', model="gpt-4-turbo-preview"): + if self.negation_manager != None: + # look up if there is a corresponding backward call in reverse_db + negation_call = self.negation_manager.get_success(forward_call) + if negation_call != None: + return negation_call + + return generate_reverse_command(forward_call, prompt, credentials, api_type, generate_mode, openai_model=model) + + def execute_api_call(self, command: str, credentials: list = None) -> int: + """Execute API call. + + Args: + command (str): API call to execute. + """ + image_id = self.docker_sandbox.create_image_from_code(command) + if image_id == None: + raise Exception("Error: failed to generate or get image id") + # Add dummy arguments to the code + command = code_add_dummy_argument(command) + result = self.docker_sandbox.create_python_sandbox(command, image_id, credentials=credentials) + + return result diff --git a/goex/exec_engine/container_utils/code_parser.py b/goex/exec_engine/container_utils/code_parser.py new file mode 100644 index 000000000..cb53b46d5 --- /dev/null +++ b/goex/exec_engine/container_utils/code_parser.py @@ -0,0 +1,406 @@ +"""For parsing code snippets and locating relevant dependencies/imports so that they could be installed to the containers' images""" + +from contextlib import contextmanager +import os +import sys +import re +import logging +import ast +from docopt import docopt +import requests +from pathlib import Path +from yarg import json2package +from yarg.exceptions import HTTPError + +REGEXP = [ + re.compile(r'^import (.+)$'), + re.compile(r'^from ((?!\.+).*?) import (?:.*)$') +] + +logging.disable(sys.maxsize) +PIPS_FOLDER_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent), "pips") +DOCKER_REQUIREMENTS_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__)).parent.parent), "docker/docker/requirements.txt") + +@contextmanager +def _open(filename=None, mode='r'): + """Open a file or ``sys.stdout`` depending on the provided filename. + + Args: + filename (str): The path to the file that should be opened. If + ``None`` or ``'-'``, ``sys.stdout`` or ``sys.stdin`` is + returned depending on the desired mode. Defaults to ``None``. + mode (str): The mode that should be used to open the file. + + Yields: + A file handle. + + """ + if not filename or filename == '-': + if not mode or 'r' in mode: + file = sys.stdin + elif 'w' in mode: + file = sys.stdout + else: + raise ValueError('Invalid mode for file: {}'.format(mode)) + else: + file = open(filename, mode) + + try: + yield file + finally: + if file not in (sys.stdin, sys.stdout): + file.close() + + +def get_all_imports( + contents): + imports = set() + raw_imports = set() + candidates = [] + ignore_errors = False + + tree = ast.parse(contents) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for subnode in node.names: + raw_imports.add(subnode.name) + elif isinstance(node, ast.ImportFrom): + raw_imports.add(node.module) + + # Clean up imports + for name in [n for n in raw_imports if n]: + # Sanity check: Name could have been None if the import + # statement was as ``from . import X`` + # Cleanup: We only want to first part of the import. + # Ex: from django.conf --> django.conf. But we only want django + # as an import. + cleaned_name, _, _ = name.partition('.') + imports.add(cleaned_name) + + packages = imports - (set(candidates) & imports) + logging.debug('Found packages: {0}'.format(packages)) + + with open(os.path.join(PIPS_FOLDER_PATH, "stdlib.txt"), "r") as f: + data = {x.strip() for x in f} + + return list(packages - data) + + +def generate_requirements_file(path, imports, symbol): + with _open(path, "w") as out_file: + logging.debug('Writing {num} requirements: {imports} to {file}'.format( + num=len(imports), + file=path, + imports=", ".join([x['name'] for x in imports]) + )) + fmt = '{name}' + symbol + '{version}' + out_file.write('\n'.join( + fmt.format(**item) if item['version'] else '{name}'.format(**item) + for item in imports) + '\n') + + +def get_imports_info( + imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): + result = [] + + # transformers most times require torch and sentencepiece + if ('transformers' in imports): + imports.extend(['torch', 'sentencepiece']) + + for item in imports: + try: + logging.warning( + 'Import named "%s" not found locally. ' + 'Trying to resolve it at the PyPI server.', + item + ) + response = requests.get( + "{0}{1}/json".format(pypi_server, item), proxies=proxy) + if response.status_code == 200: + if hasattr(response.content, 'decode'): + data = json2package(response.content.decode()) + else: + data = json2package(response.content) + elif response.status_code >= 300: + raise HTTPError(status_code=response.status_code, + reason=response.reason) + except HTTPError: + logging.warning( + 'Package "%s" does not exist or network problems', item) + continue + logging.warning( + 'Import named "%s" was resolved to "%s:%s" package (%s).\n' + 'Please, verify manually the final list of requirements.txt ' + 'to avoid possible dependency confusions.', + item, + data.name, + data.latest_release_id, + data.pypi_url + ) + result.append({'name': item, 'version': data.latest_release_id}) + return result + + +def get_locally_installed_packages(encoding=None): + packages = [] + ignore = ["tests", "_tests", "egg", "EGG", "info"] + for path in sys.path: + for root, dirs, files in os.walk(path): + for item in files: + if "top_level" in item: + item = os.path.join(root, item) + with open(item, "r", encoding=encoding) as f: + package = root.split(os.sep)[-1].split("-") + try: + top_level_modules = f.read().strip().split("\n") + except: # NOQA + # TODO: What errors do we intend to suppress here? + continue + + # filter off explicitly ignored top-level modules + # such as test, egg, etc. + filtered_top_level_modules = list() + + for module in top_level_modules: + if ( + (module not in ignore) and + (package[0] not in ignore) + ): + # append exported top level modules to the list + filtered_top_level_modules.append(module) + + version = None + if len(package) > 1: + version = package[1].replace( + ".dist", "").replace(".egg", "") + + # append package: top_level_modules pairs + # instead of top_level_module: package pairs + packages.append({ + 'name': package[0], + 'version': version, + 'exports': filtered_top_level_modules + }) + return packages + + +def get_import_local(imports, encoding=None): + local = get_locally_installed_packages() + result = [] + if ('transformers' in imports): + imports.extend(['torch', 'sentencepiece']) + for item in imports: + # search through local packages + for package in local: + # if candidate import name matches export name + # or candidate import name equals to the package name + # append it to the result + if item in package['exports'] or item == package['name']: + result.append(package) + + # removing duplicates of package/version + # had to use second method instead of the previous one, + # because we have a list in the 'exports' field + # https://stackoverflow.com/questions/9427163/remove-duplicate-dict-in-list-in-python + name_set = set() + result_unique = [] + for n, i in enumerate(result): + if i['name'] not in name_set: + result_unique.append(i) + name_set.add(i['name']) + + #result_unique = [i for n, i in enumerate(result) if i not in result[n+1:]] + + return result_unique + + +def get_pkg_names(pkgs): + """Get PyPI package names from a list of imports. + + Args: + pkgs (List[str]): List of import names. + + Returns: + List[str]: The corresponding PyPI package names. + + """ + result = set() + with open(os.path.join(PIPS_FOLDER_PATH, "mapping.txt"), "r") as f: + data = dict(x.strip().split(":") for x in f) + for pkg in pkgs: + # Look up the mapped requirement. If a mapping isn't found, + # simply use the package name. + result.add(data.get(pkg, pkg)) + # Return a sorted list for backward compatibility. + return sorted(result, key=lambda s: s.lower()) + + +def get_name_without_alias(name): + if "import " in name: + match = REGEXP[0].match(name.strip()) + if match: + name = match.groups(0)[0] + return name.partition(' as ')[0].partition('.')[0].strip() + + +def join(f): + return os.path.join(os.path.dirname(__file__), f) + + +def parse_requirements(file_): + """Parse a requirements formatted file. + + Traverse a string until a delimiter is detected, then split at said + delimiter, get module name by element index, create a dict consisting of + module:version, and add dict to list of parsed modules. + + Args: + file_: File to parse. + + Raises: + OSerror: If there's any issues accessing the file. + + Returns: + tuple: The contents of the file, excluding comments. + """ + modules = [] + # For the dependency identifier specification, see + # https://www.python.org/dev/peps/pep-0508/#complete-grammar + delim = ["<", ">", "=", "!", "~"] + + try: + f = open(file_, "r") + except OSError: + logging.error("Failed on file: {}".format(file_)) + raise + else: + try: + data = [x.strip() for x in f.readlines() if x != "\n"] + finally: + f.close() + + data = [x for x in data if x[0].isalpha()] + + for x in data: + # Check for modules w/o a specifier. + if not any([y in x for y in delim]): + modules.append({"name": x, "version": None}) + for y in x: + if y in delim: + module = x.split(y) + module_name = module[0] + module_version = module[-1].replace("=", "") + module = {"name": module_name, "version": module_version} + + if module not in modules: + modules.append(module) + + break + + return modules + + +def compare_modules(file_, imports): + """Compare modules in a file to imported modules in a project. + + Args: + file_ (str): File to parse for modules to be compared. + imports (tuple): Modules being imported in the project. + + Returns: + tuple: The modules not imported in the project, but do exist in the + specified file. + """ + modules = parse_requirements(file_) + + imports = [imports[i]["name"] for i in range(len(imports))] + modules = [modules[i]["name"] for i in range(len(modules))] + modules_not_imported = set(modules) - set(imports) + + return modules_not_imported + + +def diff(file_, imports): + """Display the difference between modules in a file and imported modules.""" # NOQA + modules_not_imported = compare_modules(file_, imports) + + logging.info( + "The following modules are in {} but do not seem to be imported: " + "{}".format(file_, ", ".join(x for x in modules_not_imported))) + + +def clean(file_, imports): + """Remove modules that aren't imported in project from file.""" + modules_not_imported = compare_modules(file_, imports) + + if len(modules_not_imported) == 0: + logging.info("Nothing to clean in " + file_) + return + + re_remove = re.compile("|".join(modules_not_imported)) + to_write = [] + + try: + f = open(file_, "r+") + except OSError: + logging.error("Failed on file: {}".format(file_)) + raise + else: + try: + for i in f.readlines(): + if re_remove.match(i) is None: + to_write.append(i) + f.seek(0) + f.truncate() + + for i in to_write: + f.write(i) + finally: + f.close() + + logging.info("Successfully cleaned up requirements in " + file_) + + +def dynamic_versioning(scheme, imports): + """Enables dynamic versioning with , or schemes.""" + if scheme == "no-pin": + imports = [{"name": item["name"], "version": ""} for item in imports] + symbol = "" + elif scheme == "gt": + symbol = ">=" + elif scheme == "compat": + symbol = "~=" + return imports, symbol + + +def extract_dependencies(contents, args = {}, path = DOCKER_REQUIREMENTS_PATH): + candidates = get_all_imports(contents) + candidates = get_pkg_names(candidates) + logging.debug("Found imports: " + ", ".join(candidates)) + pypi_server = "https://pypi.python.org/pypi/" + proxy = None + logging.debug("Getting packages information from Local/PyPI") + local = get_import_local(candidates, encoding=None) + + #commonly used enough to be worth adding + candidates.append('pickle') + # check if candidate name is found in + # the list of exported modules, installed locally + # and the package name is not in the list of local module names + # it add to difference + difference = [x for x in candidates if + # aggregate all export lists into one + # flatten the list + # check if candidate is in exports + x.lower() not in [y for x in local for y in x['exports']] + and + # check if candidate is package names + x.lower() not in [x['name'] for x in local]] + imports = local + get_imports_info(difference, + proxy=proxy, + pypi_server=pypi_server) + # sort imports based on lowercase name of package, similar to `pip freeze`. + imports = sorted(imports, key=lambda x: x['name'].lower()) + generate_requirements_file(path, imports, "==") + logging.info("Successfully saved requirements file in " + path) + return imports diff --git a/goex/exec_engine/container_utils/container_utils.py b/goex/exec_engine/container_utils/container_utils.py new file mode 100644 index 000000000..db32b854a --- /dev/null +++ b/goex/exec_engine/container_utils/container_utils.py @@ -0,0 +1,51 @@ +"""To speed up build runtimes. Previously used docker images are cached in images.json so that they could be reused""" + +import hashlib +import os +import json +from pathlib import Path + +ROOT_FOLDER_PATH = os.path.dirname(Path(os.path.realpath(__file__)).parent.parent) +IMAGES_FILE_PATH = os.path.join(ROOT_FOLDER_PATH, "docker/misc/images.json") + +def get_files_hash(*file_paths): + """Return the SHA256 hash of multiple files.""" + hasher = hashlib.sha256() + for file_path in file_paths: + with open(file_path, "rb") as f: + while chunk := f.read(4096): + hasher.update(chunk) + return hasher.hexdigest() + +def find_local_docker_image(image_hash): + hash_file_path = IMAGES_FILE_PATH + if not os.path.exists(hash_file_path): + return None + with open(hash_file_path, "r") as f: + stored_hashes = json.load(f) + if image_hash in stored_hashes: + return stored_hashes[image_hash] + else: + return None + +def save_image_hash(image_hash, image_name): + hash_file_path = IMAGES_FILE_PATH + try: + if not os.path.exists(hash_file_path): + with open(hash_file_path, "w") as f: + stored_hashes = {} + stored_hashes[image_hash] = image_name + json.dump(stored_hashes, f) + return True + else: + stored_hashes = {} + with open(hash_file_path, "r") as f: + stored_hashes = json.load(f) + with open(hash_file_path, "w") as f: + stored_hashes[image_hash] = image_name + json.dump(stored_hashes, f) + return True + except Exception as e: + print(e) + return False + diff --git a/goex/exec_engine/credentials/__init__.py b/goex/exec_engine/credentials/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/goex/exec_engine/credentials/credentials_utils.py b/goex/exec_engine/credentials/credentials_utils.py new file mode 100644 index 000000000..1a826d781 --- /dev/null +++ b/goex/exec_engine/credentials/credentials_utils.py @@ -0,0 +1,159 @@ +"""External services sometimes require user credentials for authentication. Users can add credentials to Gorilla and specify their preferred technique to pass their credentials.""" + +import os +import json +import os +import shutil +from pathlib import Path + +CREDS_FOLDER_PATH = os.path.dirname(os.path.realpath(__file__)) +CREDS_SUPPORTED_PATH = os.path.join(os.path.dirname(Path(os.path.realpath(__file__))), "supported.txt") +SECRET_STORE_PATH = os.path.join(CREDS_FOLDER_PATH, "secret_store.json") + +''' +check if the Gorilla supports the provided credential +''' +def cred_is_supported(cred): + creds_list = [] + with open(CREDS_SUPPORTED_PATH, 'r') as f: + for line in f: + creds_list.append(line.strip()) + if cred not in creds_list: + return False + return True + + +""" +Insert the service and its corresponding api key. +Example: service = "gmail", key = "SOME_API_KEY" +""" +def insert_creds(service, key, target = CREDS_FOLDER_PATH, cred_type="raw"): + #create secret store if it doesn't exist + secret_store_path = os.path.join(target, "secret_store.json") + secrets = {} + try: + with open(secret_store_path, "r") as secret_store: + secrets = json.load(secret_store) + except Exception as e: + "target folder doesn't contain secret store, creating..." + + with open(secret_store_path, "w") as secret_store: + secrets[service.lower()] = [key, cred_type] + json.dump(secrets, secret_store) + +def remove_creds(services, target = CREDS_FOLDER_PATH): + secret_store_path = os.path.join(target, "secret_store.json") + secrets = {} + try: + with open(secret_store_path, "r") as secret_store: + secrets = json.load(secret_store) + except Exception as e: + print("No previous authorization.") + return + + if(services[0]=='ALL'): + services = list(secrets.keys()) + + for service in services: + if(service.lower() in secrets): + try: + del secrets[service.lower()] + cred_path = os.path.join(target, service.lower()) + if os.path.exists(cred_path): + shutil.rmtree(cred_path) + print(f"Succesfully removed {service}.") + except Exception as e: + print(f"Could not remove credentials for {service}.") + else: + print(f"Authorization for {service} was not found.") + if(not secrets): + os.remove(secret_store_path) + else: + with open(secret_store_path, "w") as secret_store: + json.dump(secrets, secret_store) + +def list_creds(target = CREDS_FOLDER_PATH): + secret_store_path = os.path.join(target, "secret_store.json") + if not os.path.exists(secret_store_path ): + "target folder doesn't contain secret store, aborting..." + + with open(secret_store_path, "r") as secret_store: + try: + secrets = json.load(secret_store) + return secrets + except Exception as e: + print(e) + return {} + +def creds_from_prompt(prompt, target_folder, technique = "lut", llm_code_output = None): + creds_set = set(list_creds(target_folder)) + creds_set = list_creds(target_folder) + formatted_prompt = prompt.lower() + + found = [] + if technique == "lut": + for service in creds_set: + if service in formatted_prompt: + if not isinstance(creds_set[service], list) or len(creds_set[service]) != 2: + print("Error: credential for {service} in the secret store is not in the correct format".format(service=service)) + else: + cred, cred_type = creds_set[service] + found.append([service, cred, cred_type]) + return found + + elif technique == "openai": + found = [] + if llm_code_output: + generated_list = openai_generate_credentials_list("generate a list of credentials or keys I need to execute the following code {c}".format(c=llm_code_output)) + else: + generated_list = openai_generate_credentials_list("generate a list of credentials or keys I need to execute code for the following prompt {p}".format(p=prompt)) + + found = [key for key in generated_list if key in creds_set] + return found + + else: + print("technique has not been implemented yet...") + return + +def get_cred_paths(creds, target = CREDS_FOLDER_PATH): + secret_store_path = os.path.join(target, "secret_store.json") + if not os.path.exists(secret_store_path ): + "target folder doesn't contain secret store, aborting..." + + with open(secret_store_path, "r") as secret_store: + try: + secrets = json.load(secret_store) + found, missing = {}, [] + for c in creds: + if c in secrets: + found[c] = secrets[c][0] + else: + missing.append(c) + return found, missing + except Exception as e: + print(e) + return {}, [] + +def openai_generate_credentials_list(prompt): + from openai import OpenAI + client = OpenAI() + + response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[ + { + "role": "user", + "content": prompt + } + ] + ) + + return response.choices[0].message.content + +def list_supported_services(): + with open(os.path.join(CREDS_FOLDER_PATH, "supported.txt")) as f: + services = f.readlines() + for i, s in enumerate(services): + services[i] = s.replace("\n", "") + + return services diff --git a/goex/exec_engine/credentials/supported.txt b/goex/exec_engine/credentials/supported.txt new file mode 100644 index 000000000..4a81b2d09 --- /dev/null +++ b/goex/exec_engine/credentials/supported.txt @@ -0,0 +1,6 @@ +gmail +slack +spotify +dropbox +github +discord \ No newline at end of file diff --git a/goex/exec_engine/db_manager.py b/goex/exec_engine/db_manager.py new file mode 100644 index 000000000..00089a2bd --- /dev/null +++ b/goex/exec_engine/db_manager.py @@ -0,0 +1,302 @@ +from exec_engine.docker_sandbox import DockerSandbox + +""" + This module will handle all database interactions + The DBManager class is the base class for all database managers +""" + +class DBManager: + """Base class for all DB connectors. + + Attributes: + connection_config (type): JSON Config for connection. + + Methods: + connect: Establish connections to the DB + execute_db_call: Execute DB call + commit_db_calls: Commit DB calls + rollback_db_calls: Rollback DB calls + close: Close the connection to the database + + """ + + def __init__(self, connection_config): + """Initialize the DBManager. + + Args: + connection_config (dict): Configuration for connecting to the database. This can be a path for file-based databases or connection details for server-based databases. + + """ + self.connection_config = connection_config + self.docker_sandbox = None + + def connect(self): + """Establish connection to the database.""" + raise NotImplementedError + + def get_schema_as_string(self): + prompt = "" + for table_name, schema in self.schema.items(): + prompt += f"Table '{table_name}':\n" + for column in schema: + column_name, column_type, is_nullable, key, default, extra = column + prompt += f"- Column '{column_name}' of type '{column_type}'" + if is_nullable == 'NO': + prompt += ", not nullable" + if key == 'PRI': + prompt += ", primary key" + prompt += "\n" + prompt += "\n" + return prompt + + def task_to_prompt(self, task_description, forward=True): + """Format the schemas of all tables into a prompt for GPT, including a task description.""" + prompt = "" + + if self.schema == None: + raise Exception("Please connect to the database first.") + + if self.schema: + "No schema information available." + prompt += "Given the following table schemas in a sqlite database:\n\n" + prompt += self.get_schema_as_string() + + if forward: + prompt += f"Task: {task_description}\n\n" + prompt += "Based on the task, select the most appropriate table and generate an SQL command to complete the task. In the output, only include SQL code." + else: + prompt += f"SQL command: {task_description}\n\n" + prompt += "Based on the SQL command and the given table schemas, generate a reverse command to reverse the SQL command. In the output, only include SQL code." + return prompt + + def execute_db_call(self, call): + """Execute DB call. + + Args: + call (str): DB call to execute. + """ + raise NotImplementedError + + def fetch_db_call(self, call): + raise NotImplementedError + + def commit_db_calls(self): + """Commit DB calls.""" + raise NotImplementedError + + def rollback_db_calls(self): + """Rollback DB calls not committed""" + raise NotImplementedError + + def close(self): + """Close the connection to the database.""" + raise NotImplementedError + + +class SQLiteManager(DBManager): + """SQLite database manager. + + Attributes: + _sqlite_imported (bool): flag to check if sqlite3 is imported. + + Methods: + connect: Establish connections to the DB + execute_db_call: Execute SQL call + commit_db_calls: Commit SQL calls + rollback_db_calls: Rollback SQL calls + close: Close the connection to the database + """ + _sqlite_imported = False # flag to check if sqlite3 is imported + db_type = "sqlite" + TEST_CONFIG = "" # No config required to access sqlite + def __init__(self, connection_config, docker_sandbox: DockerSandbox = None): + """Initialize the SQLLiteManager. + + Args: + connection_config(str): path to the database file. + """ + if not SQLiteManager._sqlite_imported: + global sqlite3 + import sqlite3 + SQLiteManager._sqlite_imported = True + keys = connection_config.keys() + + if any(key not in keys for key in ['path']): + raise ValueError("Failed to initialize SQLite Manager due to bad configs") + + self.db_path = connection_config['path'] + if not self.db_path: + raise ValueError("Failed to initialize SQLite Manager due to missing path") + + def update_schema_info(self): + schema_info = {} + + self.cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + tables = self.cursor.fetchall() + for (table_name,) in tables: + self.cursor.execute(f"PRAGMA table_info({table_name});") + schema_info[table_name] = self.cursor.fetchall() + + self.schema = schema_info + + def connect(self): + """Establish connection to the SQLLite3 database and create a cursor.""" + self.conn = sqlite3.connect(self.db_path) + self.cursor = self.conn.cursor() + self.update_schema_info() + + + def execute_db_call(self, call): + if not self.conn: + self.connect() + try: + commands_list = [cmd.strip() for cmd in call.split(';') if cmd.strip() and not cmd.strip().startswith('--')] + for command in commands_list: + if command.upper().startswith('SELECT'): + self.cursor.execute(command) + print(self.cursor.fetchall()) + else: + self.cursor.execute(command) + self.update_schema_info() + return 0 + except Exception as e: + return 1 + + + def fetch_db_call(self, call): + if not self.conn: + self.connect() + try: + self.cursor.execute(call) + ret_val = self.cursor.fetchall() + self.update_schema_info() + return ret_val + except Exception as e: + return [] + + def commit_db_calls(self): + """Commit SQL calls.""" + if not self.conn: + self.connect() + self.conn.commit() + + def rollback_db_calls(self): + """Rollback SQL calls not committed""" + if not self.conn: + self.connect() + self.conn.rollback() + self.close() + self.connect() + + def close(self): + if self.conn: + self.cursor.close() + self.conn.close() + + +class MySQLManager(DBManager): + """MySQL database manager. + + Attributes: + _mysql_imported (bool): flag to check if pymysql is imported. + + Methods: + connect: Establish connections to the DB + execute_db_call: Execute SQL call + commit_db_calls: Commit SQL calls + rollback_db_calls: Rollback SQL calls + close: Close the connection to the database + """ + _mysql_imported = False + db_type = "mysql" + TEST_CONFIG = "{'host': '127.0.0.1', 'user': 'root', 'password': ''}\n Use Pymysql and make sure to create the database using subprocess before connection." + def __init__(self, connection_config, docker_sandbox: DockerSandbox = None): + """Initialize the MySQLManager. + + Args: + connection_config (dict): configuration for the database connection, including keys for 'user', 'password', 'host', and 'database'. + """ + if not MySQLManager._mysql_imported: + global pymysql + import pymysql + MySQLManager._mysql_imported = True + + keys = connection_config.keys() + + if any(key not in keys for key in ['host', 'user', 'password', 'database']): + raise ValueError("Failed to initialize MySQL Manager due to bad configs") + elif any([not connection_config['host'], not connection_config['user'], not connection_config['password'], not connection_config['database']]): + raise ValueError("Failed to initialize MySQL Manager due to missing configs") + + self.connection_config = { + 'host': connection_config['host'], + 'user': connection_config['user'], + 'password': connection_config['password'], + 'database': connection_config['database'], + "client_flag": pymysql.constants.CLIENT.MULTI_STATEMENTS + } + + def connect(self): + """Establish connection to the MySQL database and create a cursor.""" + self.conn = pymysql.connect(**self.connection_config) + self.cursor = self.conn.cursor() + self.update_schema_info() + + def update_schema_info(self): + schema_info = {} + + self.cursor.execute("SHOW TABLES") + tables = self.cursor.fetchall() + for (table_name,) in tables: + self.cursor.execute(f"DESCRIBE {table_name}") + schema_info[table_name] = self.cursor.fetchall() + + self.schema = schema_info + + def execute_db_call(self, call): + """Execute a SQL call using the cursor.""" + if not self.conn: + self.connect() + try: + self.cursor.execute(call) + self.update_schema_info() + return 0 + except Exception as e: + return 1 + + def fetch_db_call(self, call: str) -> list[dict]: + """Execute a SQL call and return the results. + + Args: + call (str): SQL query to execute. + + Returns: + list[dict]: A list of dictionaries representing each row in the query result. + """ + if not self.conn: + self.connect() + try: + self.cursor.execute(call) + ret_val = self.cursor.fetchall() + self.update_schema_info() + return ret_val + except Exception as e: + return [] + + def commit_db_calls(self): + """Commit SQL calls.""" + if not self.conn: + self.connect() + self.conn.commit() + + def rollback_db_calls(self): + """Rollback SQL calls not committed.""" + if not self.conn: + self.connect() + self.conn.rollback() + + def close(self): + """Close the cursor and the connection to the database.""" + if self.conn: + self.cursor.close() + self.conn.close() diff --git a/goex/exec_engine/docker_sandbox.py b/goex/exec_engine/docker_sandbox.py new file mode 100644 index 000000000..c4798c1dd --- /dev/null +++ b/goex/exec_engine/docker_sandbox.py @@ -0,0 +1,143 @@ +"""Manages Docker containers for a controlled environment.""" + +import docker +import os +from pathlib import Path + +import exec_engine.container_utils.container_utils as utils +from exec_engine.container_utils.code_parser import extract_dependencies +from exec_engine.credentials.credentials_utils import get_cred_paths, CREDS_FOLDER_PATH + +from exec_engine.utils import format_container_logs, RESTful_Type, SQL_Type, Filesystem_Type + +ROOT_FOLDER_PATH = os.path.dirname(Path(os.path.realpath(__file__)).parent) +DOCKER_FOLDER_PATH = os.path.join(ROOT_FOLDER_PATH, "docker/docker") +DOCKER_REQUIREMENTS_PATH = os.path.join(DOCKER_FOLDER_PATH, "requirements.txt") +DOCKERFILE_PATH = os.path.join(DOCKER_FOLDER_PATH, "dockerfile") +DOCKER_EXECUTION_PATH = os.path.join(DOCKER_FOLDER_PATH, "python_executor.py") + +MYSQL_DOCKER_FOLDER_PATH = os.path.join(ROOT_FOLDER_PATH, "docker/mysql_docker") +MYSQL_DOCKER_REQUIREMENTS_PATH = os.path.join(MYSQL_DOCKER_FOLDER_PATH, "requirements.txt") +MYSQL_DOCKERFILE_PATH = os.path.join(MYSQL_DOCKER_FOLDER_PATH, "dockerfile") +MYSQL_DOCKER_EXECUTION_PATH = os.path.join(MYSQL_DOCKER_FOLDER_PATH, "python_executor.py") + +def get_docker_paths(docker_folder_path): + requirements_path = os.path.join(docker_folder_path, "requirements.txt") + dockerfile_path = os.path.join(docker_folder_path, "dockerfile") + execution_path = os.path.join(docker_folder_path, "python_executor.py") + + return requirements_path, dockerfile_path, execution_path + +class DockerSandbox: + def __init__(self, client_config = {}): + self.client = None + self.auto_save_image = True + self.auto_remove = True + + if not client_config: + try: + self.client = docker.from_env() + except Exception as e: + print("Error: {error}.\nIf you haven't already, please install Docker https://docs.docker.com/get-docker/".format(error=e)) + else: + try: + self.client = docker.DockerClient(**client_config) + except Exception as e: + print("Unable to initialize a docker client based on client_config {client_config}. Encountered an error with message {error}".format(client_config=client_config, error=e)) + + + def create_sandbox(self): + # Create a new Docker container with a basic image + return self.client.containers.run("ubuntu", detach=True) + + + def create_python_sandbox(self, code, image_id, credentials=None, attached_volume = None): + volumes = [] + + if credentials: + paths, not_found = get_cred_paths(credentials, CREDS_FOLDER_PATH) + for service in paths: + host_path = paths[service] + container_path = "/sandbox/credentials/" + service + volumes.append(host_path + ":" + container_path) + + else: + volumes = { + # path on your machine/host + os.path.abspath("credentials"): { + "bind": "/sandbox/credentials", # path inside the container + "mode": "rw", + } + } + if attached_volume: + volumes[attached_volume] = { + "bind": "/sandbox/test_dir", + "mode": "ro", + } + + try: + container = self.client.containers.run( + image_id, + command = + ['python', 'python_executor.py', 'code_execute'], + environment = { + "CODE": code + }, + volumes=volumes, + stderr = True, + stdout=True, + detach=True, + ) + container.wait() + docker_out, docker_debug = format_container_logs(container) + if self.auto_remove: + container.remove() + + except Exception as e: + print("Failure occured inside client.containers.run with the following error message:", e) + return None + + return {"output": docker_out, "debug": docker_debug} + + def create_image_from_code(self, code, api_type=RESTful_Type): + if api_type == SQL_Type: + docker_folder_path = MYSQL_DOCKER_FOLDER_PATH + else: + docker_folder_path = DOCKER_FOLDER_PATH + + requirements_path, dockerfile_path, execution_path = get_docker_paths(docker_folder_path) + try: + extract_dependencies(code, path=requirements_path) + image_hash = utils.get_files_hash(dockerfile_path, requirements_path, execution_path) + image_id = utils.find_local_docker_image(image_hash) + except Exception as e: + print("ERROR:, ", e) + return + + # Run the container using the image specified if exists, else pull the image from docker hub + try: + if image_id: + self.client.images.get(image_id) + except Exception as e: + try: + # Use the low-level API to stream the pull response + low_level_client = docker.APIClient() + low_level_client.pull(image_id, stream=True, decode=True) + self.client.images.get(image_id) + except: + image_id = None + + try: + if not image_id: + image_id = self.client.images.build(path=docker_folder_path)[0].short_id + if self.auto_save_image: + utils.save_image_hash(image_hash, image_id) + except Exception as e: + print("Unable to build docker image, returned with error: {error}".format(error=e)) + return None + + return image_id + + def delete_sandbox(self, container): + container.stop() + container.remove() diff --git a/goex/exec_engine/fs_manager.py b/goex/exec_engine/fs_manager.py new file mode 100644 index 000000000..a25e28e0e --- /dev/null +++ b/goex/exec_engine/fs_manager.py @@ -0,0 +1,182 @@ +""" + This module will handle all filesystem interactions + The FSManager class is the base class for all filesystem managers +""" +import os +import subprocess + +class FSManager: + """Base class for all FS operations. + + Attributes: + fs_path (type): path to the fs location. + + Methods: + execute: Execute command + commit: Commit SQL calls + revert: "Revert changes to previous commit through git LFS + initialize_git_lfs: Initialize the current directory as a git lfs repo if it isn't already + """ + def __init__(self, fs_path=None, git_init=True): + """Initialize the FSManager. + + Args: + fs_path (str): path to the fs path. Default is CWD + """ + if not fs_path: + self.fs_path = os.getcwd() + else: + self.fs_path = os.path.abspath(fs_path) + if not os.path.exists(self.fs_path) or not os.path.isdir(self.fs_path): + raise Exception("Please provide a valid directory") + + self.is_git_repo = os.path.exists(os.path.join(self.fs_path, '.git')) + self.git_init = git_init + + def execute(self, command: str, display=False): + """Execute command. + + Args: + command (str): Command to execute. + """ + if display: + return subprocess.call(command, shell=True, cwd=self.fs_path) + return subprocess.call(command, shell=True, cwd=self.fs_path, stdout=subprocess.DEVNULL) + + def commit(self, message='Auto-commit via FSManager', clean=True): + """Commit all current changes through git LFS""" + try: + self.execute(f'git add .') # Stage all changes + self.execute(f'git commit -m "{message}"') # Commit with a message + if clean and not self.is_git_repo: + self.execute('rm -rf .git') # Remove git once commit happens to save space + except Exception as e: + print(f"Error during commit: {e}") + + def revert(self, clean=True): + """Revert changes to previous commit through git LFS""" + try: + self.execute('git clean -fd') + self.execute('git reset --hard HEAD') + if clean and not self.is_git_repo: + self.execute('rm -rf .git') # Remove git once revert happens to save space + except Exception as e: + print(f"Error during revert: {e}") + + def initialize_version_control(self): + """Initialize the current directory as a git lfs repo if it isn't already.""" + if self.git_init: + if not os.path.exists(os.path.join(self.fs_path, '.git')): + try: + self.execute('git init') # Initialize git repository + if self._exceed_directory_size(os.getcwd()): + self.execute('git lfs install') # Initialize git LFS + print("Initialized current directory as a Git LFS repository.") + self.execute('git add .') + self.execute("git commit --allow-empty -n -m init") + except Exception as e: + print(f"Error during Git initialization: {e}") + else: + # check to see if there are uncommitted changes + uncommitted = self._check_uncommitted_changes() + if uncommitted: + raise Exception("Please commit or stash all changes before executing FS commands") + + + def task_to_prompt(self, task_description, forward=True): + """ + Formats a prompt for GPT-4 including the directory tree and a user task description. + + :param directory_tree: A nested dictionary representing the directory structure. + :param task_description: A string describing the task to be performed. + :return: A formatted string prompt. + """ + + tree_summary = self._get_directory_tree() + + prompt = ( + "Given the following project directory structure:\n" + f"{tree_summary}\n\n" + ) + + # Format the full prompt including the directory structure and the task description + if forward: + prompt += ( + "Perform the following task:\n" + f"{task_description}\n\n" + "Generate the command to achieve this and only include the shell command as a shell codeblock in the response, " + ) + else: + prompt += ( + "Given this shell command:\n" + f"{task_description}\n\n" + "Generate the reverse command to reverse the given shell command and only include the shell command as a shell codeblock in the response, " + ) + prompt += "make sure to not change CWD:" + return prompt + + def _get_directory_tree(self): + """ + Creates a nested dictionary that represents the folder structure of self.fs_path. + """ + summary_lines = [] + + # Ensure the self.fs_path ends with a separator to correctly calculate depth + if not self.fs_path.endswith(os.sep): + self.fs_path += os.sep + + for root, dirs, files in os.walk(self.fs_path): + if '.git' in dirs: + dirs.remove('.git') + # Calculate the depth based on the difference between root and self.fs_path + depth = root.replace(self.fs_path, '').count(os.sep) + indent = " " * depth # Four spaces per indentation level + + # Extract the last part of the root path to get the current directory name + dirname = os.path.basename(root.rstrip(os.sep)) + if depth > 0: # Avoid including the self.fs_path itself as a directory line + summary_lines.append(f"{indent}+ {dirname}/") + + for name in sorted(dirs + files): + if name in dirs: + # Directories will be processed in subsequent iterations of os.walk + continue + else: + # Append files directly + summary_lines.append(f"{indent} - {name}") + + return "\n".join(summary_lines) + + def _exceed_directory_size(self, path, size_limit=200): + """size_limit is measured in MB""" + total_size = 0 + for dirpath, dirnames, filenames in os.walk(path): + for f in filenames: + fp = os.path.join(dirpath, f) + # skip if it is symbolic link + if not os.path.islink(fp): + total_size += os.path.getsize(fp) + if total_size >= size_limit * 1024 * 1024: + return True + return False + + def _check_uncommitted_changes(self): + # Ensure the path exists and is a directory + if not os.path.isdir(self.fs_path): + print(f"The path {self.fs_path} is not a valid directory.") + return + + try: + result = subprocess.run(["git", "status", "--porcelain"], cwd=self.fs_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + # Check if the output is empty. If it's not, there are uncommitted changes. + if result.stdout.strip(): + print("There are uncommitted changes or untracked files.") + return True + else: + return False + + except Exception as e: + print(f"An error occurred: {e}") + return True + diff --git a/goex/exec_engine/negation_manager.py b/goex/exec_engine/negation_manager.py new file mode 100644 index 000000000..75cc13bf6 --- /dev/null +++ b/goex/exec_engine/negation_manager.py @@ -0,0 +1,78 @@ +import os +import json + +""" +NegationAPIPairManager Interface +""" +class NegationAPIPairManager: + def __init__(self): + return None + + def get_success(self, forward_call): + # return a single backward call that have previously been reported successful + # this backward call will be used to perform the reversion if requested + raise NotImplementedError + + def get_failure(self, forward_call): + # return a list of backward calls that have previously been reported unsuccessful + raise NotImplementedError + + def insert_log(self, forward_call, backward_call, result): + # insert a new entry or log to the negation manager + raise NotImplementedError + +""" +Simple NegationManager with JSON based key-value pair storage +""" +class NaiveNegationAPIPairManager(NegationAPIPairManager): + def __init__(self, reverse_log_json_path): + self.reverse_log_path = os.path.join(reverse_log_json_path, "negation_log.json") + + def get_success(self, forward_call): + logs = {} + try: + with open(self.reverse_log_path, "r") as reverse_log: + logs = json.load(reverse_log) + if forward_call in logs: + if "true" in logs[forward_call]: + return logs[forward_call]["true"][0] + + except Exception as e: + pass + + return None + + def get_failure(self, forward_call): + logs = {} + try: + with open(self.reverse_log_path, "r") as reverse_log: + logs = json.load(reverse_log) + if forward_call in logs: + if "false" in logs[forward_call]: + return logs[forward_call]["false"] + + except Exception as e: + pass + + return None + + def insert_log(self, forward_call, backward_call, result): + logs = {} + try: + with open(self.reverse_log_path, "r") as reverse_log: + logs = json.load(reverse_log) + + except Exception as e: + pass + + with open(self.reverse_log_path, "w") as reverse_log: + if forward_call not in logs: + logs[forward_call] = {} + + result = str(result).lower() + if result in logs[forward_call]: + logs[forward_call][result].append(backward_call) + else: + logs[forward_call][result] = [backward_call] + + json.dump(logs, reverse_log) diff --git a/goex/exec_engine/pipeline.py b/goex/exec_engine/pipeline.py new file mode 100644 index 000000000..aa280d87f --- /dev/null +++ b/goex/exec_engine/pipeline.py @@ -0,0 +1,311 @@ +import requests +import os +from openai import OpenAI, APIError +from collections import defaultdict +from exec_engine.credentials.credentials_utils import * +from exec_engine.utils import SQL_Type, Filesystem_Type, RESTful_Type + +import re + +python_pattern = r"```python\n(.*?)\n```" +sql_pattern = r"```sql\n(.*?)\n```" +bash_pattern = r"```(?:bash|shell|sh)\n(.*?)\n```" + +INSTRUCTION_TEXT = """ + You are an assistant that outputs executable Python code that perform what the user requests. + It is important that you only return one and only one code block with all the necessary imports inside ```python and nothing else. + The code block should print the output(s) when appropriate. + + If the action can't be successfully completed, throw an exception + + This is what the user requests: {request}\n + """ + +REVERSE_INSTRUCTION_TEXT = """ + Given an action and a Python code block that performs that action from the user, + you are an assistant that outputs executable Python code that perform the REVERSE (can make more/new API calls if needed) of what's been done. + + It is important that the REVERSE code only revert the changes if any and nothing else, and that you only return + one and only one code block with all the necessary imports inside ```python and nothing else. + + The code block should print the output(s) when appropriate. + + If the action can't be successfully completed, throw an exception + + This is the action: {prompt} + This is the action code block: {forward_call} + + If no revert action exists, return a python code with a print statement explaining why so. + """ + +FUNCTION_IN_CONTEXT_TEXT = """ + You are an assistant that outputs executable Python code according to a given list of python functions that perform what the user requests. + It is important that you only return one and only one code block with all the necessary imports inside ```python and nothing else. + The code block should print the output(s) when appropriate. + + If the action can't be successfully completed, throw an exception + + Don't change the available functions listed below except for feeding parameters. + + Only use the functions defined below, don't add new functions. + + Here is a list of python functions available:\n + {functions}\n + + This is what the user requests: {request}\n + """ + +REVERSE_FUNCTION_IN_CONTEXT_TEXT = """ + Given a list of available functions, an action and a Python code block that performs that action from the user, + you are an assistant that outputs executable Python code using the available functions provided that perform the REVERSE (can make more/new API calls if needed) of what's been done. + + It is important that the REVERSE code only revert the changes if any and nothing else, and that you only return + one and only one code block with all the necessary imports inside ```python and nothing else. + + The code block should print the output(s) when appropriate. + + Don't change the available functions listed below except for feeding parameters. + + Only use the functions defined below, don't add new functions. + + If the action can't be successfully completed, throw an exception + + Here is a list of python functions available:\n + {functions}\n + + This is the action: {prompt} + This is the action code block: {forward_call} + + If no revert action exists, return a python code with a print statement explaining why so. + """ + +FUNCTION_CALLING_NATIVE_TEXT = """ + {request} + """ + +REVERSE_FUNCTION_CALLING_NATIVE_TEXT = """ + Given an action and a Python code block that performs that action from the user, + perform the REVERSE of what's been done. + + This is the action: {prompt} + This is the action code block: {forward_call} + """ + +def read_available_functions(): + FUNCTION_DIR = "./function/" + avail_functions = {} + functions = os.listdir(FUNCTION_DIR) + for function in functions: + if not function.endswith(".py"): + continue + with open(FUNCTION_DIR + function, "r") as file: + avail_functions[function.replace(".py", "")] = file.read() + return avail_functions + + +def convert_to_function_call(function_call_list): + if type(function_call_list) == dict: + function_call_list = [function_call_list] + execution_list = [] + for function_call in function_call_list: + for key, value in function_call.items(): + execution_list.append( + f"{key}({','.join([f'{k}={repr(v)}' for k,v in json.loads(value).items()])})" + ) + return execution_list + + +def generate_command(content, credentials = None, api_type=RESTful_Type, generate_mode='default', openai_model="gpt-4-turbo-preview"): + client = OpenAI() + + # defaults to getting the key using os.environ.get("OPENAI_API_KEY") + # if you saved the key under a different environment variable name, you can do something like: + # client = OpenAI( + # api_key=os.environ.get("CUSTOM_ENV_NAME"), + # ) + + if api_type == SQL_Type: + prompt = content + pattern = sql_pattern + elif api_type == Filesystem_Type: + prompt = content + pattern = bash_pattern + elif api_type == RESTful_Type or True: + if generate_mode == 'default': + prompt = INSTRUCTION_TEXT.format(request=content) + elif generate_mode == 'function_in_context': + avail_functions = read_available_functions() + prompt_functions = "" + for idx, function in enumerate(list(avail_functions.values())): + prompt_functions += "<>: " + str(function) + "\n" + prompt = FUNCTION_IN_CONTEXT_TEXT.format(functions=prompt_functions, request=content) + elif generate_mode == 'function_calling_native': + prompt = FUNCTION_CALLING_NATIVE_TEXT.format(request=content) + else: + raise Exception("Error: {} is not a supported argument for generate_mode, please select again from 'default', 'function_in_context', 'function_calling_native'" + .format(generate_mode)) + + if credentials: + prompt += generate_credentials_prompt_info(credentials) + + else: + prompt += "Equally importantly, try to do so without requiring any API key. If and only if one is really needed, give the code block assuming those keys are provided." + + pattern = python_pattern + + if generate_mode == 'function_calling_native': + function_calls = json.load(open('./function/function.json')) + response = client.chat.completions.create( + model=openai_model, + messages=[ + { + "role": "user", + "content": prompt + } + ], + tools=function_calls, + tool_choice=None, + ) + + if response.choices[0].message.tool_calls == None: + raise Exception("Unable able to retreive the relevant command to perform the action from Function Calling.\n" + + "Try again with a different prompt or generate_mode setting") + + output = [ + {func_call.function.name: func_call.function.arguments} + for func_call in response.choices[0].message.tool_calls + ] + assert len(output) == 1 + output = output[0] + python_code = open(f"./function/{list(output.keys())[0]}.py", 'r').read() + output = convert_to_function_call(output)[0] + output = "```python\n" + python_code + "\n" + output + "\n```" + else: + response = client.chat.completions.create( + model=openai_model, + messages=[ + { + "role": "user", + "content": prompt + } + ] + ) + output = response.choices[0].message.content + + #print(output) #Intentionally left here for future debugging purpose + matches = re.search(pattern, output, re.DOTALL) + if matches: + code = matches.group(1) + return code + +def generate_reverse_command(forward_call, prompt, credentials=None, api_type=RESTful_Type, generate_mode='default', openai_model="gpt-4-turbo-preview"): + client = OpenAI() + + if api_type == RESTful_Type: + + if generate_mode == 'default': + reverse_prompt = REVERSE_INSTRUCTION_TEXT.format(prompt=prompt, forward_call=forward_call) + elif generate_mode == 'function_in_context': + avail_functions = read_available_functions() + prompt_functions = "" + for idx, function in enumerate(list(avail_functions.values())): + prompt_functions += "<>: " + str(function) + "\n" + reverse_prompt = REVERSE_FUNCTION_IN_CONTEXT_TEXT.format(prompt=prompt, forward_call=forward_call, functions=prompt_functions) + elif generate_mode == 'function_calling_native': + reverse_prompt = REVERSE_FUNCTION_CALLING_NATIVE_TEXT.format(prompt=prompt, forward_call=forward_call) + else: + raise Exception("Error: {} is not a supported argument for generate_mode, please select again from 'default', 'function_in_context', 'function_calling_native'" + .format(generate_mode)) + + if credentials: + reverse_prompt += generate_credentials_prompt_info(credentials) + else: + reverse_prompt += "Equally importantly, try to do so without requiring any API key. If and only if one is really needed, give the code block assuming those keys are provided." + else: + raise Exception("generate_reverse_command not supported for execution type other than REST") + + if generate_mode == 'function_calling_native': + function_calls = json.load(open('./function/function.json')) + response = client.chat.completions.create( + model=openai_model, + messages=[ + { + "role": "user", + "content": reverse_prompt + } + ], + tools=function_calls, + tool_choice=None, + ) + output = [ + {func_call.function.name: func_call.function.arguments} + for func_call in response.choices[0].message.tool_calls + ] + assert len(output) == 1 + output = output[0] + python_code = open(f"./function/{list(output.keys())[0]}.py", 'r').read() + output = convert_to_function_call(output)[0] + output = "```python\n" + python_code + "\n" + output + "\n```" + + else: + response = client.chat.completions.create( + model=openai_model, + temperature=0.3, + messages=[ + { + "role": "user", + "content": reverse_prompt + } + ] + ) + output = response.choices[0].message.content + + #print(output) #Intentionally left here for future debugging purpose + matches = re.search(python_pattern, output, re.DOTALL) + if matches: + code = matches.group(1) + return code + +def prompt_execute(engine, prompt, services=None, creds=None, max_attempt=3, model="gpt-4-turbo-preview"): + ret = defaultdict(list) + for _ in range(max_attempt): + forward_call, backward_call = engine.gen_api_pair(prompt, api_type=RESTful_Type, credentials=creds, model=model) + response = engine.api_executor.execute_api_call(forward_call, services) + + if response and response['output']: + ret['output'].append(response['output']) + return ret, forward_call, backward_call + elif response and response['debug']: + ret['debug'].append(response['debug']) + + engine._add_api_reverse_to_queue(RESTful_Type, (forward_call, backward_call)) + + return ret, forward_call, None + + +def generate_credentials_prompt_info(credentials): + #credential comes in a [service_name, value, cred_type] format + token_in_path = [] + raw_key = [] + try: + for service_name, value, cred_type in credentials: + if cred_type == "path": + token_in_path.append([service_name, value]) + elif cred_type == "raw": + raw_key.append([service_name, value]) + except: + raise Exception("Error: credentials have to be passed in as a list of [service_name, value, cred_type] pairs") + + if token_in_path != []: + cred_paths = {} + for service_name, value in token_in_path: + prefix = "./credentials/" + service_name + "/" + cred_paths[service_name] = [prefix + file_name for file_name in os.listdir(value)] + + return "The credentials (such as API keys) are stored in the following paths: {keys_list}. Open the file(s) to access them".format(keys_list=cred_paths) + + if raw_key != []: + return "Additionally, these api keys are available for the following services {key_list}".format( + key_list=" ,".join("{name}={cred}".format(name=name, cred=cred) for name,cred in raw_key)) + + diff --git a/goex/exec_engine/pips/mapping.txt b/goex/exec_engine/pips/mapping.txt new file mode 100644 index 000000000..cbc747fd9 --- /dev/null +++ b/goex/exec_engine/pips/mapping.txt @@ -0,0 +1,1152 @@ +AFQ:pyAFQ +AG_fft_tools:agpy +ANSI:pexpect +Adafruit:Adafruit_Libraries +App:Zope2 +Asterisk:py_Asterisk +BB_jekyll_hook:bitbucket_jekyll_hook +Banzai:Banzai_NGS +BeautifulSoupTests:BeautifulSoup +BioSQL:biopython +BuildbotStatusShields:BuildbotEightStatusShields +ComputedAttribute:ExtensionClass +constraint:python-constraint +Crypto:pycryptodome +Cryptodome:pycryptodomex +FSM:pexpect +FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper +functional:pyfunctional +GeoBaseMain:GeoBasesDev +GeoBases:GeoBasesDev +Globals:Zope2 +HelpSys:Zope2 +IPython:ipython +Kittens:astro_kittens +Levenshtein:python_Levenshtein +Lifetime:Zope2 +MethodObject:ExtensionClass +MySQLdb:MySQL-python +OFS:Zope2 +OpenGL:PyOpenGL +OpenSSL:pyOpenSSL +PIL:Pillow +Products:Zope2 +PyWCSTools:astLib +Pyxides:astro_pyxis +QtCore:PySide +S3:s3cmd +SCons:pystick +Shared:Zope2 +Signals:Zope2 +Stemmer:PyStemmer +Testing:Zope2 +TopZooTools:topzootools +TreeDisplay:DocumentTemplate +WorkingWithDocumentConversion:aspose_pdf_java_for_python +ZPublisher:Zope2 +ZServer:Zope2 +ZTUtils:Zope2 +aadb:auto_adjust_display_brightness +abakaffe:abakaffe_cli +abiosgaming:abiosgaming.py +abiquo:abiquo_api +abl:abl.cssprocessor +abl:abl.robot +abl:abl.util +abl:abl.vpath +abo:abo_generator +abris_transform:abris +abstract:abstract.jwrotator +abu:abu.admin +ac_flask:AC_Flask_HipChat +acg:anikom15 +acme:acme.dchat +acme:acme.hello +acted:acted.projects +action:ActionServer +actionbar:actionbar.panel +activehomed:afn +activepapers:ActivePapers.Py +address_book:address_book_lansry +adi:adi.commons +adi:adi.devgen +adi:adi.fullscreen +adi:adi.init +adi:adi.playlist +adi:adi.samplecontent +adi:adi.slickstyle +adi:adi.suite +adi:adi.trash +adict:aDict2 +aditam:aditam.agent +aditam:aditam.core +adiumsh:adium_sh +adjector:AdjectorClient +adjector:AdjectorTracPlugin +adkit:Banner_Ad_Toolkit +admin_tools:django_admin_tools +adminishcategories:adminish_categories +adminsortable:django_admin_sortable +adspygoogle:adspygoogle.adwords +advancedcaching:agtl +adytum:Adytum_PyMonitor +affinitic:affinitic.docpyflakes +affinitic:affinitic.recipe.fakezope2eggs +affinitic:affinitic.simplecookiecuttr +affinitic:affinitic.verifyinterface +affinitic:affinitic.zamqp +afpy:afpy.xap +agatesql:agate_sql +ageliaco:ageliaco.recipe.csvconfig +agent_http:agent.http +agora:Agora_Client +agora:Agora_Fountain +agora:Agora_Fragment +agora:Agora_Planner +agora:Agora_Service_Provider +agoraplex:agoraplex.themes.sphinx +agsci:agsci.blognewsletter +agx:agx.core +agx:agx.dev +agx:agx.generator.buildout +agx:agx.generator.dexterity +agx:agx.generator.generator +agx:agx.generator.plone +agx:agx.generator.pyegg +agx:agx.generator.sql +agx:agx.generator.uml +agx:agx.generator.zca +agx:agx.transform.uml2fs +agx:agx.transform.xmi2uml +aimes:aimes.bundle +aimes:aimes.skeleton +aio:aio.app +aio:aio.config +aio:aio.core +aio:aio.signals +aiohs2:aio_hs2 +aioroutes:aio_routes +aios3:aio_s3 +airbrake:airbrake_flask +airship:airship_icloud +airship:airship_steamcloud +airflow:apache-airflow +akamai:edgegrid_python +alation:alation_api +alba_client:alba_client_python +alburnum:alburnum_maas_client +alchemist:alchemist.audit +alchemist:alchemist.security +alchemist:alchemist.traversal +alchemist:alchemist.ui +alchemyapi:alchemyapi_python +alerta:alerta_server +alexandria_upload:Alexandria_Upload_Utils +alibaba:alibaba_python_sdk +aliyun:aliyun_python_sdk +aliyuncli:alicloudcli +aliyunsdkacs:aliyun_python_sdk_acs +aliyunsdkbatchcompute:aliyun_python_sdk_batchcompute +aliyunsdkbsn:aliyun_python_sdk_bsn +aliyunsdkbss:aliyun_python_sdk_bss +aliyunsdkcdn:aliyun_python_sdk_cdn +aliyunsdkcms:aliyun_python_sdk_cms +aliyunsdkcore:aliyun_python_sdk_core +aliyunsdkcrm:aliyun_python_sdk_crm +aliyunsdkcs:aliyun_python_sdk_cs +aliyunsdkdrds:aliyun_python_sdk_drds +aliyunsdkecs:aliyun_python_sdk_ecs +aliyunsdkess:aliyun_python_sdk_ess +aliyunsdkft:aliyun_python_sdk_ft +aliyunsdkmts:aliyun_python_sdk_mts +aliyunsdkocs:aliyun_python_sdk_ocs +aliyunsdkoms:aliyun_python_sdk_oms +aliyunsdkossadmin:aliyun_python_sdk_ossadmin +aliyunsdkr-kvstore:aliyun_python_sdk_r_kvstore +aliyunsdkram:aliyun_python_sdk_ram +aliyunsdkrds:aliyun_python_sdk_rds +aliyunsdkrisk:aliyun_python_sdk_risk +aliyunsdkros:aliyun_python_sdk_ros +aliyunsdkslb:aliyun_python_sdk_slb +aliyunsdksts:aliyun_python_sdk_sts +aliyunsdkubsms:aliyun_python_sdk_ubsms +aliyunsdkyundun:aliyun_python_sdk_yundun +allattachments:AllAttachmentsMacro +allocine:allocine_wrapper +allowedsites:django_allowedsites +alm:alm.solrindex +aloft:aloft.py +alpacalib:alpaca +alphabetic:alphabetic_simple +alphasms:alphasms_client +altered:altered.states +alterootheme:alterootheme.busycity +alterootheme:alterootheme.intensesimplicity +alterootheme:alterootheme.lazydays +alurinium:alurinium_image_processing +alxlib:alx +amara3:amara3_iri +amara3:amara3_xml +amazon:AmazonAPIWrapper +amazon:python_amazon_simple_product_api +ambikesh1349-1:ambikesh1349_1 +ambilight:AmbilightParty +amifs:amifs_core +amiorganizer:ami_organizer +amitu:amitu.lipy +amitu:amitu_putils +amitu:amitu_websocket_client +amitu:amitu_zutils +amltlearn:AMLT_learn +amocrm:amocrm_api +amqpdispatcher:amqp_dispatcher +amqpstorm:AMQP_Storm +analytics:analytics_python +analyzedir:AnalyzeDirectory +ancientsolutions:ancientsolutions_crypttools +anderson_paginator:anderson.paginator +android_clean_app:android_resource_remover +anel_power_control:AnelPowerControl +angus:angus_sdk_python +annalist_root:Annalist +annogesiclib:ANNOgesic +ansible-role-apply:ansible_role_apply +ansibledebugger:ansible_playbook_debugger +ansibledocgen:ansible_docgen +ansibleflow:ansible_flow +ansibleinventorygrapher:ansible_inventory_grapher +ansiblelint:ansible_lint +ansiblerolesgraph:ansible_roles_graph +ansibletools:ansible_tools +anthill:anthill.exampletheme +anthill:anthill.skinner +anthill:anthill.tal.macrorenderer +anthrax:AnthraxDojoFrontend +anthrax:AnthraxHTMLInput +anthrax:AnthraxImage +antisphinx:antiweb +antispoofing:antispoofing.evaluation +antlr4:antlr4_python2_runtime +antlr4:antlr4_python3_runtime +antlr4:antlr4_python_alt +anybox:anybox.buildbot.openerp +anybox:anybox.nose.odoo +anybox:anybox.paster.odoo +anybox:anybox.paster.openerp +anybox:anybox.recipe.sysdeps +anybox:anybox.scripts.odoo +apiclient:google_api_python_client +apitools:google_apitools +apm:arpm +app_data:django_appdata +appconf:django_appconf +appd:AppDynamicsDownloader +appd:AppDynamicsREST +appdynamics_bindeps:appdynamics_bindeps_linux_x64 +appdynamics_bindeps:appdynamics_bindeps_linux_x86 +appdynamics_bindeps:appdynamics_bindeps_osx_x64 +appdynamics_proxysupport:appdynamics_proxysupport_linux_x64 +appdynamics_proxysupport:appdynamics_proxysupport_linux_x86 +appdynamics_proxysupport:appdynamics_proxysupport_osx_x64 +appium:Appium_Python_Client +appliapps:applibase +appserver:broadwick +archetypes:archetypes.kss +archetypes:archetypes.multilingual +archetypes:archetypes.schemaextender +arm:ansible_role_manager +armor:armor_api +armstrong:armstrong.apps.related_content +armstrong:armstrong.apps.series +armstrong:armstrong.cli +armstrong:armstrong.core.arm_access +armstrong:armstrong.core.arm_layout +armstrong:armstrong.core.arm_sections +armstrong:armstrong.core.arm_wells +armstrong:armstrong.dev +armstrong:armstrong.esi +armstrong:armstrong.hatband +armstrong:armstrong.templates.standard +armstrong:armstrong.utils.backends +armstrong:armstrong.utils.celery +arstecnica:arstecnica.raccoon.autobahn +arstecnica:arstecnica.sqlalchemy.async +article-downloader:article_downloader +artifactcli:artifact_cli +arvados:arvados_python_client +arvados_cwl:arvados_cwl_runner +arvnodeman:arvados_node_manager +asana_to_github:AsanaToGithub +asciibinary:AsciiBinaryConverter +asd:AdvancedSearchDiscovery +askbot:askbot_tuan +askbot:askbot_tuanpa +asnhistory:asnhistory_redis +aspen_jinja2_renderer:aspen_jinja2 +aspen_tornado_engine:aspen_tornado +asprise_ocr_api:asprise_ocr_sdk_python_api +aspy:aspy.refactor_imports +aspy:aspy.yaml +asterisk:asterisk_ami +asts:add_asts +asymmetricbase:asymmetricbase.enum +asymmetricbase:asymmetricbase.fields +asymmetricbase:asymmetricbase.logging +asymmetricbase:asymmetricbase.utils +asyncirc:asyncio_irc +asyncmongoorm:asyncmongoorm_je +asyncssh:asyncssh_unofficial +athletelist:athletelistyy +atm:automium +atmosphere:atmosphere_python_client +atom:gdata +atomic:AtomicWrite +atomisator:atomisator.db +atomisator:atomisator.enhancers +atomisator:atomisator.feed +atomisator:atomisator.indexer +atomisator:atomisator.outputs +atomisator:atomisator.parser +atomisator:atomisator.readers +atreal:atreal.cmfeditions.unlocker +atreal:atreal.filestorage.common +atreal:atreal.layouts +atreal:atreal.mailservices +atreal:atreal.massloader +atreal:atreal.monkeyplone +atreal:atreal.override.albumview +atreal:atreal.richfile.preview +atreal:atreal.richfile.qualifier +atreal:atreal.usersinout +atsim:atsim.potentials +attractsdk:attract_sdk +audio:audio.bitstream +audio:audio.coders +audio:audio.filters +audio:audio.fourier +audio:audio.frames +audio:audio.lp +audio:audio.psychoacoustics +audio:audio.quantizers +audio:audio.shrink +audio:audio.wave +aufrefer:auf_refer +auslfe:auslfe.formonline.content +auspost:auspost_apis +auth0:auth0_python +auth_server_client:AuthServerClient +authorize:AuthorizeSauce +authzpolicy:AuthzPolicyPlugin +autobahn:autobahn_rce +avatar:geonode_avatar +awebview:android_webview +azure:azure_common +azure:azure_mgmt_common +azure:azure_mgmt_compute +azure:azure_mgmt_network +azure:azure_mgmt_nspkg +azure:azure_mgmt_resource +azure:azure_mgmt_storage +azure:azure_nspkg +azure:azure_servicebus +azure:azure_servicemanagement_legacy +azure:azure_storage +b2gcommands:b2g_commands +b2gperf:b2gperf_v1.3 +b2gperf:b2gperf_v1.4 +b2gperf:b2gperf_v2.0 +b2gperf:b2gperf_v2.1 +b2gperf:b2gperf_v2.2 +b2gpopulate:b2gpopulate_v1.3 +b2gpopulate:b2gpopulate_v1.4 +b2gpopulate:b2gpopulate_v2.0 +b2gpopulate:b2gpopulate_v2.1 +b2gpopulate:b2gpopulate_v2.2 +b3j0f:b3j0f.annotation +b3j0f:b3j0f.aop +b3j0f:b3j0f.conf +b3j0f:b3j0f.sync +b3j0f:b3j0f.utils +babel:Babel +babelglade:BabelGladeExtractor +backplane:backplane2_pyclient +backport_abcoll:backport_collections +backports:backports.functools_lru_cache +backports:backports.inspect +backports:backports.pbkdf2 +backports:backports.shutil_get_terminal_size +backports:backports.socketpair +backports:backports.ssl +backports:backports.ssl_match_hostname +backports:backports.statistics +badgekit:badgekit_api_client +badlinks:BadLinksPlugin +bael:bael.project +baidu:baidupy +balrog:buildtools +baluhn:baluhn_redux +bamboo:bamboo.pantrybell +bamboo:bamboo.scaffold +bamboo:bamboo.setuptools_version +bamboo:bamboo_data +bamboo:bamboo_server +bambu:bambu_codemirror +bambu:bambu_dataportability +bambu:bambu_enqueue +bambu:bambu_faq +bambu:bambu_ffmpeg +bambu:bambu_grids +bambu:bambu_international +bambu:bambu_jwplayer +bambu:bambu_minidetect +bambu:bambu_navigation +bambu:bambu_notifications +bambu:bambu_payments +bambu:bambu_pusher +bambu:bambu_saas +bambu:bambu_sites +banana:Bananas +banana:banana.maya +bang:bangtext +barcode:barcode_generator +bark:bark_ssg +barking_owl:BarkingOwl +bart:bart_py +basalt:basalt_tasks +base62:base_62 +basemap:basemap_Jim +bash:bash_toolbelt +bashutils:Python_Bash_Utils +basic_http:BasicHttp +basil:basil_daq +batchapps:azure_batch_apps +bcrypt:python_bcrypt +beaker:Beaker +beetsplug:beets +begin:begins +benchit:bench_it +beproud:beproud.utils +bfillings:burrito_fillings +bigjob:BigJob +billboard:billboard.py +binstar_build_client:anaconda_build +binstar_client:anaconda_client +biocommons:biocommons.dev +birdhousebuilder:birdhousebuilder.recipe.conda +birdhousebuilder:birdhousebuilder.recipe.docker +birdhousebuilder:birdhousebuilder.recipe.redis +birdhousebuilder:birdhousebuilder.recipe.supervisor +blender26-meshio:pymeshio +bootstrap:BigJob +borg:borg.localrole +bow:bagofwords +bpdb:bpython +bqapi:bisque_api +braces:django_braces +briefscaster:briefs_caster +brisa_media_server/plugins:brisa_media_server_plugins +brkt_requests:brkt_sdk +broadcastlogging:broadcast_logging +brocadetool:brocade_tool +bronto:bronto_python +brownie:Brownie +browsermobproxy:browsermob_proxy +brubeckmysql:brubeck_mysql +brubeckoauth:brubeck_oauth +brubeckservice:brubeck_service +brubeckuploader:brubeck_uploader +bs4:beautifulsoup4 +bson:pymongo +bst:bst.pygasus.core +bst:bst.pygasus.datamanager +bst:bst.pygasus.demo +bst:bst.pygasus.i18n +bst:bst.pygasus.resources +bst:bst.pygasus.scaffolding +bst:bst.pygasus.security +bst:bst.pygasus.session +bst:bst.pygasus.wsgi +btable:btable_py +btapi:bananatag_api +btceapi:btce_api +btcebot:btce_bot +btsync:btsync.py +buck:buck.pprint +bud:bud.nospam +budy:budy_api +buffer:buffer_alpaca +buggd:bug.gd +bugle:bugle_sites +bugspots:bug_spots +bugzilla:python_bugzilla +bugzscout:bugzscout_py +buildTools:ajk_ios_buildTools +buildnotifylib:BuildNotify +buildout:buildout.bootstrap +buildout:buildout.disablessl +buildout:buildout.dumppickedversions +buildout:buildout.dumppickedversions2 +buildout:buildout.dumprequirements +buildout:buildout.eggnest +buildout:buildout.eggscleaner +buildout:buildout.eggsdirectories +buildout:buildout.eggtractor +buildout:buildout.extensionscripts +buildout:buildout.locallib +buildout:buildout.packagename +buildout:buildout.recipe.isolation +buildout:buildout.removeaddledeggs +buildout:buildout.requirements +buildout:buildout.sanitycheck +buildout:buildout.sendpickedversions +buildout:buildout.threatlevel +buildout:buildout.umask +buildout:buildout.variables +buildslave:buildbot_slave +builtins:pies2overrides +bumper:bumper_lib +bumple:bumple_downloader +bundesliga:bundesliga_cli +bundlemaker:bundlemanager +burpui:burp_ui +busyflow:busyflow.pivotal +buttercms-django:buttercms_django +buzz:buzz_python_client +bvc:buildout_versions_checker +bvggrabber:bvg_grabber +byond:BYONDTools +bzETL:Bugzilla_ETL +bzlib:bugzillatools +bzrlib:bzr +bzrlib:bzr_automirror +bzrlib:bzr_bash_completion +bzrlib:bzr_colo +bzrlib:bzr_killtrailing +bzrlib:bzr_pqm +c2c:c2c.cssmin +c2c:c2c.recipe.closurecompile +c2c:c2c.recipe.cssmin +c2c:c2c.recipe.jarfile +c2c:c2c.recipe.msgfmt +c2c:c2c.recipe.pkgversions +c2c:c2c.sqlalchemy.rest +c2c:c2c.versions +c2c_recipe_facts:c2c.recipe.facts +cabalgata:cabalgata_silla_de_montar +cabalgata:cabalgata_zookeeper +cache_utils:django_cache_utils +captcha:django_recaptcha +cartridge:Cartridge +cassandra:cassandra_driver +cassandralauncher:CassandraLauncher +cc42:42qucc +cerberus:Cerberus +cfnlint:cfn-lint +chameleon:Chameleon +charmtools:charm_tools +chef:PyChef +chip8:c8d +cjson:python_cjson +classytags:django_classy_tags +cloghandler:ConcurrentLogHandler +clonevirtualenv:virtualenv_clone +cloud-insight:al_cloudinsight +cloud_admin:adminapi +cloudservers:python_cloudservers +clusterconsole:cerebrod +clustersitter:cerebrod +cms:django_cms +colander:ba_colander +colors:ansicolors +compile:bf_lc3 +compose:docker_compose +compressor:django_compressor +concurrent:futures +configargparse:ConfigArgParse +configparser:pies2overrides +contracts:PyContracts +coordination:BigJob +copyreg:pies2overrides +corebio:weblogo +couchapp:Couchapp +couchdb:CouchDB +couchdbcurl:couchdb_python_curl +courseradownloader:coursera_dl +cow:cow_framework +creole:python_creole +creoleparser:Creoleparser +crispy_forms:django_crispy_forms +cronlog:python_crontab +crontab:python_crontab +ctff:tff +cups:pycups +curator:elasticsearch_curator +curl:pycurl +daemon:python_daemon +dare:DARE +dateutil:python_dateutil +dawg:DAWG +deb822:python_debian +debian:python_debian +decouple:python-decouple +demo:webunit +demosongs:PySynth +deployer:juju_deployer +depot:filedepot +devtools:tg.devtools +dgis:2gis +dhtmlparser:pyDHTMLParser +digitalocean:python_digitalocean +discord:discord.py +distribute_setup:ez_setup +distutils2:Distutils2 +django:Django +django_hstore:amitu_hstore +djangobower:django_bower +djcelery:django_celery +djkombu:django_kombu +djorm_pgarray:djorm_ext_pgarray +dns:dnspython +docgen:ansible_docgenerator +docker:docker_py +dogpile:dogpile.cache +dogpile:dogpile.core +dogshell:dogapi +dot_parser:pydot +dot_parser:pydot2 +dot_parser:pydot3k +dotenv:python-dotenv +dpkt:dpkt_fix +dsml:python_ldap +durationfield:django_durationfield +dzclient:datazilla +easybuild:easybuild_framework +editor:python_editor +elasticluster:azure_elasticluster +elasticluster:azure_elasticluster_current +elftools:pyelftools +elixir:Elixir +em:empy +emlib:empy +enchant:pyenchant +encutils:cssutils +engineio:python_engineio +enum:enum34 +ephem:pyephem +errorreporter:abl.errorreporter +esplot:beaker_es_plot +example:adrest +examples:tweepy +ez_setup:pycassa +fabfile:Fabric +fabric:Fabric +faker:Faker +fdpexpect:pexpect +fedora:python_fedora +fias:ailove_django_fias +fiftyone_degrees:51degrees_mobile_detector +five:five.customerize +five:five.globalrequest +five:five.intid +five:five.localsitemanager +five:five.pt +flasher:android_flasher +flask:Flask +flask_frozen:Frozen_Flask +flask_redis:Flask_And_Redis +flaskext:Flask_Bcrypt +flvscreen:vnc2flv +followit:django_followit +forge:pyforge +formencode:FormEncode +formtools:django_formtools +fourch:4ch +franz:allegrordf +freetype:freetype_py +frontmatter:python_frontmatter +ftpcloudfs:ftp_cloudfs +funtests:librabbitmq +fuse:fusepy +fuzzy:Fuzzy +gabbi:tiddlyweb +gen_3dwallet:3d_wallet_generator +gendimen:android_gendimen +genshi:Genshi +geohash:python_geohash +geonode:GeoNode +geoserver:gsconfig +geraldo:Geraldo +getenv:django_getenv +geventwebsocket:gevent_websocket +gflags:python_gflags +git:GitPython +github:PyGithub +github3:github3.py +gitpy:git_py +globusonline:globusonline_transfer_api_client +google:protobuf +googleapiclient:google_api_python_client +grace-dizmo:grace_dizmo +grammar:anovelmous_grammar +grapheneapi:graphenelib +greplin:scales +gridfs:pymongo +grokcore:grokcore.component +gslib:gsutil +hamcrest:PyHamcrest +harpy:HARPy +hawk:PyHawk_with_a_single_extra_commit +haystack:django_haystack +hgext:mercurial +hggit:hg_git +hglib:python_hglib +ho:pisa +hola:amarokHola +hoover:Hoover +hostlist:python_hostlist +html:pies2overrides +htmloutput:nosehtmloutput +http:pies2overrides +hvad:django_hvad +hydra:hydra-core +i99fix:199Fix +igraph:python_igraph +imdb:IMDbPY +impala:impyla +inmemorystorage:ambition_inmemorystorage +ipaddress:backport_ipaddress +jaraco:jaraco.timing +jaraco:jaraco.util +jinja2:Jinja2 +jiracli:jira_cli +johnny:johnny_cache +jpgrid:python_geohash +jpiarea:python_geohash +jpype:JPype1 +jpypex:JPype1 +jsonfield:django_jsonfield +jstools:aino_jstools +jupyterpip:jupyter_pip +jwt:PyJWT +kazoo:asana_kazoo +kernprof:line_profiler +keyczar:python_keyczar +keyedcache:django_keyedcache +keystoneclient:python_keystoneclient +kickstarter:kickstart +krbv:krbV +kss:kss.core +kuyruk:Kuyruk +langconv:AdvancedLangConv +lava:lava_utils_interface +lazr:lazr.authentication +lazr:lazr.restfulclient +lazr:lazr.uri +ldap:python_ldap +ldaplib:adpasswd +ldapurl:python_ldap +ldif:python_ldap +lib2or3:2or3 +lib3to2:3to2 +libaito:Aito +libbe:bugs_everywhere +libbucket:bucket +libcloud:apache_libcloud +libfuturize:future +libgenerateDS:generateDS +libmproxy:mitmproxy +libpasteurize:future +libsvm:7lk_ocr_deploy +lisa:lisa_server +loadingandsaving:aspose_words_java_for_python +locust:locustio +logbook:Logbook +logentries:buildbot_status_logentries +logilab:logilab_mtconverter +machineconsole:cerebrod +machinesitter:cerebrod +magic:python_magic +mako:Mako +manifestparser:ManifestDestiny +marionette:marionette_client +markdown:Markdown +marks:pytest_marks +markupsafe:MarkupSafe +mavnative:pymavlink +memcache:python_memcached +metacomm:AllPairs +metaphone:Metafone +metlog:metlog_py +mezzanine:Mezzanine +migrate:sqlalchemy_migrate +mimeparse:python_mimeparse +minitage:minitage.paste +minitage:minitage.recipe.common +missingdrawables:android_missingdrawables +mixfiles:PySynth +mkfreq:PySynth +mkrst_themes:2lazy2rest +mockredis:mockredispy +modargs:python_modargs +model_utils:django_model_utils +models:asposebarcode +models:asposestorage +moksha:moksha.common +moksha:moksha.hub +moksha:moksha.wsgi +moneyed:py_moneyed +mongoalchemy:MongoAlchemy +monthdelta:MonthDelta +mopidy:Mopidy +mopytools:MoPyTools +mptt:django_mptt +mpv:python-mpv +mrbob:mr.bob +msgpack:msgpack_python +mutations:aino_mutations +mws:amazon_mws +mysql:mysql_connector_python +native_tags:django_native_tags +ndg:ndg_httpsclient +nereid:trytond_nereid +nested:baojinhuan +nester:Amauri +nester:abofly +nester:bssm_pythonSig +novaclient:python_novaclient +oauth2_provider:alauda_django_oauth +oauth2client:oauth2client +odf:odfpy +ometa:Parsley +openid:python_openid +opensearchsdk:ali_opensearch +oslo_i18n:oslo.i18n +oslo_serialization:oslo.serialization +oslo_utils:oslo.utils +oss:alioss +oss:aliyun_python_sdk_oss +oss:aliyunoss +output:cashew +owslib:OWSLib +packetdiag:nwdiag +paho:paho_mqtt +paintstore:django_paintstore +parler:django_parler +past:future +paste:PasteScript +path:forked_path +path:path.py +patricia:patricia-trie +paver:Paver +peak:ProxyTypes +picasso:anderson.picasso +picklefield:django-picklefield +pilot:BigJob +pivotal:pivotal_py +play_wav:PySynth +playhouse:peewee +plivoxml:plivo +plone:plone.alterego +plone:plone.api +plone:plone.app.blob +plone:plone.app.collection +plone:plone.app.content +plone:plone.app.contentlisting +plone:plone.app.contentmenu +plone:plone.app.contentrules +plone:plone.app.contenttypes +plone:plone.app.controlpanel +plone:plone.app.customerize +plone:plone.app.dexterity +plone:plone.app.discussion +plone:plone.app.event +plone:plone.app.folder +plone:plone.app.i18n +plone:plone.app.imaging +plone:plone.app.intid +plone:plone.app.layout +plone:plone.app.linkintegrity +plone:plone.app.locales +plone:plone.app.lockingbehavior +plone:plone.app.multilingual +plone:plone.app.portlets +plone:plone.app.querystring +plone:plone.app.redirector +plone:plone.app.registry +plone:plone.app.relationfield +plone:plone.app.textfield +plone:plone.app.theming +plone:plone.app.users +plone:plone.app.uuid +plone:plone.app.versioningbehavior +plone:plone.app.viewletmanager +plone:plone.app.vocabularies +plone:plone.app.widgets +plone:plone.app.workflow +plone:plone.app.z3cform +plone:plone.autoform +plone:plone.batching +plone:plone.behavior +plone:plone.browserlayer +plone:plone.caching +plone:plone.contentrules +plone:plone.dexterity +plone:plone.event +plone:plone.folder +plone:plone.formwidget.namedfile +plone:plone.formwidget.recurrence +plone:plone.i18n +plone:plone.indexer +plone:plone.intelligenttext +plone:plone.keyring +plone:plone.locking +plone:plone.memoize +plone:plone.namedfile +plone:plone.outputfilters +plone:plone.portlet.collection +plone:plone.portlet.static +plone:plone.portlets +plone:plone.protect +plone:plone.recipe.zope2install +plone:plone.registry +plone:plone.resource +plone:plone.resourceeditor +plone:plone.rfc822 +plone:plone.scale +plone:plone.schema +plone:plone.schemaeditor +plone:plone.session +plone:plone.stringinterp +plone:plone.subrequest +plone:plone.supermodel +plone:plone.synchronize +plone:plone.theme +plone:plone.transformchain +plone:plone.uuid +plone:plone.z3cform +plonetheme:plonetheme.barceloneta +png:pypng +polymorphic:django_polymorphic +postmark:python_postmark +powerprompt:bash_powerprompt +prefetch:django-prefetch +printList:AndrewList +progressbar:progressbar2 +progressbar:progressbar33 +provider:django_oauth2_provider +puresasl:pure_sasl +pwiz:peewee +pxssh:pexpect +py7zlib:pylzma +pyAMI:pyAMI_core +pyarsespyder:arsespyder +pyasdf:asdf +pyaspell:aspell_python_ctypes +pybb:pybbm +pybloomfilter:pybloomfiltermmap +pyccuracy:Pyccuracy +pyck:PyCK +pycrfsuite:python_crfsuite +pydispatch:PyDispatcher +pygeolib:pygeocoder +pygments:Pygments +pygraph:python_graph_core +pyjon:pyjon.utils +pyjsonrpc:python_jsonrpc +pykka:Pykka +pylogo:PyLogo +pylons:adhocracy_Pylons +pymagic:libmagic +pymycraawler:Amalwebcrawler +pynma:AbakaffeNotifier +pyphen:Pyphen +pyrimaa:AEI +pysideuic:PySide +pysqlite2:adhocracy_pysqlite +pysqlite2:pysqlite +pysynth_b:PySynth +pysynth_beeper:PySynth +pysynth_c:PySynth +pysynth_d:PySynth +pysynth_e:PySynth +pysynth_p:PySynth +pysynth_s:PySynth +pysynth_samp:PySynth +pythongettext:python_gettext +pythonjsonlogger:python_json_logger +pyutilib:PyUtilib +pyximport:Cython +qs:qserve +quadtree:python_geohash +queue:future +quickapi:django_quickapi +quickunit:nose_quickunit +rackdiag:nwdiag +radical:radical.pilot +radical:radical.utils +reStructuredText:Zope2 +readability:readability_lxml +readline:gnureadline +recaptcha_works:django_recaptcha_works +relstorage:RelStorage +reportapi:django_reportapi +reprlib:pies2overrides +requests:Requests +requirements:requirements_parser +rest_framework:djangorestframework +restclient:py_restclient +retrial:async_retrial +reversion:django_reversion +rhaptos2:rhaptos2.common +robot:robotframework +robots:django_robots +rosdep2:rosdep +rsbackends:RSFile +ruamel:ruamel.base +s2repoze:pysaml2 +saga:saga_python +saml2:pysaml2 +samtranslator:aws-sam-translator +sass:libsass +sassc:libsass +sasstests:libsass +sassutils:libsass +sayhi:alex_sayhi +scalrtools:scalr +scikits:scikits.talkbox +scratch:scratchpy +screen:pexpect +scss:pyScss +sdict:dict.sorted +sdk_updater:android_sdk_updater +sekizai:django_sekizai +sendfile:pysendfile +serial:pyserial +setuputils:astor +shapefile:pyshp +shapely:Shapely +sika:ahonya_sika +singleton:pysingleton +sittercommon:cerebrod +skbio:scikit_bio +sklearn:scikit_learn +slack:slackclient +slugify:unicode_slugify +slugify:python-slugify +smarkets:smk_python_sdk +snappy:ctypes_snappy +socketio:python-socketio +socketserver:pies2overrides +sockjs:sockjs_tornado +socks:SocksiPy_branch +solr:solrpy +solution:Solution +sorl:sorl_thumbnail +south:South +sphinx:Sphinx +sphinx_pypi_upload:ATD_document +sphinxcontrib:sphinxcontrib_programoutput +sqlalchemy:SQLAlchemy +src:atlas +src:auto_mix_prep +stats_toolkit:bw_stats_toolkit +statsd:dogstatsd_python +stdnum:python_stdnum +stoneagehtml:StoneageHTML +storages:django_storages +stubout:mox +suds:suds_jurko +swiftclient:python_swiftclient +sx:pisa +tabix:pytabix +taggit:django_taggit +tasksitter:cerebrod +tastypie:django_tastypie +teamcity:teamcity_messages +telebot:pyTelegramBotAPI +telegram:python-telegram-bot +tempita:Tempita +tenjin:Tenjin +termstyle:python_termstyle +test:pytabix +thclient:treeherder_client +threaded_multihost:django_threaded_multihost +threecolor:3color_Press +tidylib:pytidylib +tkinter:future +tlw:3lwg +toredis:toredis_fork +tornadoredis:tornado_redis +tower_cli:ansible_tower_cli +trac:Trac +tracopt:Trac +translation_helper:android_localization_helper +treebeard:django_treebeard +trytond:trytond_stock +tsuru:tsuru_circus +tvrage:python_tvrage +tw2:tw2.core +tw2:tw2.d3 +tw2:tw2.dynforms +tw2:tw2.excanvas +tw2:tw2.forms +tw2:tw2.jit +tw2:tw2.jqplugins.flot +tw2:tw2.jqplugins.gritter +tw2:tw2.jqplugins.ui +tw2:tw2.jquery +tw2:tw2.sqla +twisted:Twisted +twitter:python_twitter +txclib:transifex_client +u115:115wangpan +unidecode:Unidecode +universe:ansible_universe +usb:pyusb +useless:useless.pipes +userpass:auth_userpass +utilities:automakesetup.py +utkik:aino_utkik +uwsgidecorators:uWSGI +valentine:ab +validate:configobj +version:chartio +virtualenvapi:ar_virtualenv_api +vyatta:brocade_plugins +webdav:Zope2 +weblogolib:weblogo +webob:WebOb +websocket:websocket_client +webtest:WebTest +werkzeug:Werkzeug +wheezy:wheezy.caching +wheezy:wheezy.core +wheezy:wheezy.http +wikklytext:tiddlywebwiki +winreg:future +winrm:pywinrm +workflow:Alfred_Workflow +wsmeext:WSME +wtforms:WTForms +wtfpeewee:wtf_peewee +xdg:pyxdg +xdist:pytest_xdist +xmldsig:pysaml2 +xmlenc:pysaml2 +xmlrpc:pies2overrides +xmpp:xmpppy +xstatic:XStatic_Font_Awesome +xstatic:XStatic_jQuery +xstatic:XStatic_jquery_ui +yaml:PyYAML +z3c:z3c.autoinclude +z3c:z3c.caching +z3c:z3c.form +z3c:z3c.formwidget.query +z3c:z3c.objpath +z3c:z3c.pt +z3c:z3c.relationfield +z3c:z3c.traverser +z3c:z3c.zcmlhook +zmq:pyzmq +zopyx:zopyx.textindexng3 diff --git a/goex/exec_engine/pips/stdlib.txt b/goex/exec_engine/pips/stdlib.txt new file mode 100644 index 000000000..ec03d2594 --- /dev/null +++ b/goex/exec_engine/pips/stdlib.txt @@ -0,0 +1,1785 @@ +_abc +abc +aifc +_aix_support +antigravity +argparse +array +_ast +ast +asynchat +_asyncio +asyncio +asyncio.base_events +asyncio.base_futures +asyncio.base_subprocess +asyncio.base_tasks +asyncio.constants +asyncio.coroutines +asyncio.events +asyncio.exceptions +asyncio.format_helpers +asyncio.futures +asyncio.locks +asyncio.log +asyncio.__main__ +asyncio.proactor_events +asyncio.protocols +asyncio.queues +asyncio.runners +asyncio.selector_events +asyncio.sslproto +asyncio.staggered +asyncio.streams +asyncio.subprocess +asyncio.tasks +asyncio.threads +asyncio.transports +asyncio.trsock +asyncio.unix_events +asyncio.windows_events +asyncio.windows_utils +asyncore +atexit +audioop +base64 +bdb +binascii +binhex +_bisect +bisect +_blake2 +_bootlocale +_bootsubprocess +builtins +_bz2 +bz2 +calendar +cgi +cgitb +chunk +cmath +cmd +code +_codecs +codecs +_codecs_cn +_codecs_hk +_codecs_iso2022 +_codecs_jp +_codecs_kr +_codecs_tw +codeop +_collections +collections +_collections_abc +collections.abc +colorsys +_compat_pickle +compileall +_compression +concurrent +concurrent.futures +concurrent.futures._base +concurrent.futures.process +concurrent.futures.thread +configparser +contextlib +_contextvars +contextvars +copy +copyreg +cProfile +_crypt +crypt +_csv +csv +_ctypes +ctypes +ctypes._aix +ctypes._endian +ctypes.macholib +ctypes.macholib.dyld +ctypes.macholib.dylib +ctypes.macholib.framework +_ctypes_test +ctypes.test +ctypes.test.__main__ +ctypes.test.test_anon +ctypes.test.test_array_in_pointer +ctypes.test.test_arrays +ctypes.test.test_as_parameter +ctypes.test.test_bitfields +ctypes.test.test_buffers +ctypes.test.test_bytes +ctypes.test.test_byteswap +ctypes.test.test_callbacks +ctypes.test.test_cast +ctypes.test.test_cfuncs +ctypes.test.test_checkretval +ctypes.test.test_delattr +ctypes.test.test_errno +ctypes.test.test_find +ctypes.test.test_frombuffer +ctypes.test.test_funcptr +ctypes.test.test_functions +ctypes.test.test_incomplete +ctypes.test.test_init +ctypes.test.test_internals +ctypes.test.test_keeprefs +ctypes.test.test_libc +ctypes.test.test_loading +ctypes.test.test_macholib +ctypes.test.test_memfunctions +ctypes.test.test_numbers +ctypes.test.test_objects +ctypes.test.test_parameters +ctypes.test.test_pep3118 +ctypes.test.test_pickling +ctypes.test.test_pointers +ctypes.test.test_prototypes +ctypes.test.test_python_api +ctypes.test.test_random_things +ctypes.test.test_refcounts +ctypes.test.test_repr +ctypes.test.test_returnfuncptrs +ctypes.test.test_simplesubclasses +ctypes.test.test_sizes +ctypes.test.test_slicing +ctypes.test.test_stringptr +ctypes.test.test_strings +ctypes.test.test_struct_fields +ctypes.test.test_structures +ctypes.test.test_unaligned_structures +ctypes.test.test_unicode +ctypes.test.test_values +ctypes.test.test_varsize_struct +ctypes.test.test_win32 +ctypes.test.test_wintypes +ctypes.util +ctypes.wintypes +_curses +curses +curses.ascii +curses.has_key +_curses_panel +curses.panel +curses.textpad +dataclasses +_datetime +datetime +_dbm +dbm +dbm.dumb +dbm.gnu +dbm.ndbm +_decimal +decimal +difflib +dis +distutils +distutils.archive_util +distutils.bcppcompiler +distutils.ccompiler +distutils.cmd +distutils.command +distutils.command.bdist +distutils.command.bdist_dumb +distutils.command.bdist_msi +distutils.command.bdist_packager +distutils.command.bdist_rpm +distutils.command.bdist_wininst +distutils.command.build +distutils.command.build_clib +distutils.command.build_ext +distutils.command.build_py +distutils.command.build_scripts +distutils.command.check +distutils.command.clean +distutils.command.config +distutils.command.install +distutils.command.install_data +distutils.command.install_egg_info +distutils.command.install_headers +distutils.command.install_lib +distutils.command.install_scripts +distutils.command.register +distutils.command.sdist +distutils.command.upload +distutils.config +distutils.core +distutils.cygwinccompiler +distutils.debug +distutils.dep_util +distutils.dir_util +distutils.dist +distutils.errors +distutils.extension +distutils.fancy_getopt +distutils.filelist +distutils.file_util +distutils.log +distutils.msvc9compiler +distutils._msvccompiler +distutils.msvccompiler +distutils.spawn +distutils.sysconfig +distutils.tests +distutils.tests.support +distutils.tests.test_archive_util +distutils.tests.test_bdist +distutils.tests.test_bdist_dumb +distutils.tests.test_bdist_msi +distutils.tests.test_bdist_rpm +distutils.tests.test_bdist_wininst +distutils.tests.test_build +distutils.tests.test_build_clib +distutils.tests.test_build_ext +distutils.tests.test_build_py +distutils.tests.test_build_scripts +distutils.tests.test_check +distutils.tests.test_clean +distutils.tests.test_cmd +distutils.tests.test_config +distutils.tests.test_config_cmd +distutils.tests.test_core +distutils.tests.test_cygwinccompiler +distutils.tests.test_dep_util +distutils.tests.test_dir_util +distutils.tests.test_dist +distutils.tests.test_extension +distutils.tests.test_filelist +distutils.tests.test_file_util +distutils.tests.test_install +distutils.tests.test_install_data +distutils.tests.test_install_headers +distutils.tests.test_install_lib +distutils.tests.test_install_scripts +distutils.tests.test_log +distutils.tests.test_msvc9compiler +distutils.tests.test_msvccompiler +distutils.tests.test_register +distutils.tests.test_sdist +distutils.tests.test_spawn +distutils.tests.test_sysconfig +distutils.tests.test_text_file +distutils.tests.test_unixccompiler +distutils.tests.test_upload +distutils.tests.test_util +distutils.tests.test_version +distutils.tests.test_versionpredicate +distutils.text_file +distutils.unixccompiler +distutils.util +distutils.version +distutils.versionpredicate +doctest +_dummy_thread +dummy_threading +_elementtree +email +email.base64mime +email.charset +email.contentmanager +email._encoded_words +email.encoders +email.errors +email.feedparser +email.generator +email.header +email.headerregistry +email._header_value_parser +email.iterators +email.message +email.mime +email.mime.application +email.mime.audio +email.mime.base +email.mime.image +email.mime.message +email.mime.multipart +email.mime.nonmultipart +email.mime.text +email._parseaddr +email.parser +email.policy +email._policybase +email.quoprimime +email.utils +encodings +encodings.aliases +encodings.ascii +encodings.base64_codec +encodings.big5 +encodings.big5hkscs +encodings.bz2_codec +encodings.charmap +encodings.cp037 +encodings.cp1006 +encodings.cp1026 +encodings.cp1125 +encodings.cp1140 +encodings.cp1250 +encodings.cp1251 +encodings.cp1252 +encodings.cp1253 +encodings.cp1254 +encodings.cp1255 +encodings.cp1256 +encodings.cp1257 +encodings.cp1258 +encodings.cp273 +encodings.cp424 +encodings.cp437 +encodings.cp500 +encodings.cp720 +encodings.cp737 +encodings.cp775 +encodings.cp850 +encodings.cp852 +encodings.cp855 +encodings.cp856 +encodings.cp857 +encodings.cp858 +encodings.cp860 +encodings.cp861 +encodings.cp862 +encodings.cp863 +encodings.cp864 +encodings.cp865 +encodings.cp866 +encodings.cp869 +encodings.cp874 +encodings.cp875 +encodings.cp932 +encodings.cp949 +encodings.cp950 +encodings.euc_jis_2004 +encodings.euc_jisx0213 +encodings.euc_jp +encodings.euc_kr +encodings.gb18030 +encodings.gb2312 +encodings.gbk +encodings.hex_codec +encodings.hp_roman8 +encodings.hz +encodings.idna +encodings.iso2022_jp +encodings.iso2022_jp_1 +encodings.iso2022_jp_2 +encodings.iso2022_jp_2004 +encodings.iso2022_jp_3 +encodings.iso2022_jp_ext +encodings.iso2022_kr +encodings.iso8859_1 +encodings.iso8859_10 +encodings.iso8859_11 +encodings.iso8859_13 +encodings.iso8859_14 +encodings.iso8859_15 +encodings.iso8859_16 +encodings.iso8859_2 +encodings.iso8859_3 +encodings.iso8859_4 +encodings.iso8859_5 +encodings.iso8859_6 +encodings.iso8859_7 +encodings.iso8859_8 +encodings.iso8859_9 +encodings.johab +encodings.koi8_r +encodings.koi8_t +encodings.koi8_u +encodings.kz1048 +encodings.latin_1 +encodings.mac_arabic +encodings.mac_centeuro +encodings.mac_croatian +encodings.mac_cyrillic +encodings.mac_farsi +encodings.mac_greek +encodings.mac_iceland +encodings.mac_latin2 +encodings.mac_roman +encodings.mac_romanian +encodings.mac_turkish +encodings.mbcs +encodings.oem +encodings.palmos +encodings.ptcp154 +encodings.punycode +encodings.quopri_codec +encodings.raw_unicode_escape +encodings.rot_13 +encodings.shift_jis +encodings.shift_jis_2004 +encodings.shift_jisx0213 +encodings.tis_620 +encodings.undefined +encodings.unicode_escape +encodings.utf_16 +encodings.utf_16_be +encodings.utf_16_le +encodings.utf_32 +encodings.utf_32_be +encodings.utf_32_le +encodings.utf_7 +encodings.utf_8 +encodings.utf_8_sig +encodings.uu_codec +encodings.zlib_codec +ensurepip +ensurepip._bundled +ensurepip.__main__ +ensurepip._uninstall +enum +errno +faulthandler +fcntl +filecmp +fileinput +fnmatch +formatter +fractions +_frozen_importlib +_frozen_importlib_external +ftplib +_functools +functools +__future__ +gc +_gdbm +genericpath +getopt +getpass +gettext +glob +graphlib +grp +gzip +_hashlib +hashlib +_heapq +heapq +hmac +html +html.entities +html.parser +http +http.client +http.cookiejar +http.cookies +http.server +idlelib +idlelib.autocomplete +idlelib.autocomplete_w +idlelib.autoexpand +idlelib.browser +idlelib.calltip +idlelib.calltip_w +idlelib.codecontext +idlelib.colorizer +idlelib.config +idlelib.configdialog +idlelib.config_key +idlelib.debugger +idlelib.debugger_r +idlelib.debugobj +idlelib.debugobj_r +idlelib.delegator +idlelib.dynoption +idlelib.editor +idlelib.filelist +idlelib.format +idlelib.grep +idlelib.help +idlelib.help_about +idlelib.history +idlelib.hyperparser +idlelib.idle +idlelib.idle_test +idlelib.idle_test.htest +idlelib.idle_test.mock_idle +idlelib.idle_test.mock_tk +idlelib.idle_test.template +idlelib.idle_test.test_autocomplete +idlelib.idle_test.test_autocomplete_w +idlelib.idle_test.test_autoexpand +idlelib.idle_test.test_browser +idlelib.idle_test.test_calltip +idlelib.idle_test.test_calltip_w +idlelib.idle_test.test_codecontext +idlelib.idle_test.test_colorizer +idlelib.idle_test.test_config +idlelib.idle_test.test_configdialog +idlelib.idle_test.test_config_key +idlelib.idle_test.test_debugger +idlelib.idle_test.test_debugger_r +idlelib.idle_test.test_debugobj +idlelib.idle_test.test_debugobj_r +idlelib.idle_test.test_delegator +idlelib.idle_test.test_editmenu +idlelib.idle_test.test_editor +idlelib.idle_test.test_filelist +idlelib.idle_test.test_format +idlelib.idle_test.test_grep +idlelib.idle_test.test_help +idlelib.idle_test.test_help_about +idlelib.idle_test.test_history +idlelib.idle_test.test_hyperparser +idlelib.idle_test.test_iomenu +idlelib.idle_test.test_macosx +idlelib.idle_test.test_mainmenu +idlelib.idle_test.test_multicall +idlelib.idle_test.test_outwin +idlelib.idle_test.test_parenmatch +idlelib.idle_test.test_pathbrowser +idlelib.idle_test.test_percolator +idlelib.idle_test.test_pyparse +idlelib.idle_test.test_pyshell +idlelib.idle_test.test_query +idlelib.idle_test.test_redirector +idlelib.idle_test.test_replace +idlelib.idle_test.test_rpc +idlelib.idle_test.test_run +idlelib.idle_test.test_runscript +idlelib.idle_test.test_scrolledlist +idlelib.idle_test.test_search +idlelib.idle_test.test_searchbase +idlelib.idle_test.test_searchengine +idlelib.idle_test.test_sidebar +idlelib.idle_test.test_squeezer +idlelib.idle_test.test_stackviewer +idlelib.idle_test.test_statusbar +idlelib.idle_test.test_text +idlelib.idle_test.test_textview +idlelib.idle_test.test_tooltip +idlelib.idle_test.test_tree +idlelib.idle_test.test_undo +idlelib.idle_test.test_warning +idlelib.idle_test.test_window +idlelib.idle_test.test_zoomheight +idlelib.iomenu +idlelib.macosx +idlelib.__main__ +idlelib.mainmenu +idlelib.multicall +idlelib.outwin +idlelib.parenmatch +idlelib.pathbrowser +idlelib.percolator +idlelib.pyparse +idlelib.pyshell +idlelib.query +idlelib.redirector +idlelib.replace +idlelib.rpc +idlelib.run +idlelib.runscript +idlelib.scrolledlist +idlelib.search +idlelib.searchbase +idlelib.searchengine +idlelib.sidebar +idlelib.squeezer +idlelib.stackviewer +idlelib.statusbar +idlelib.textview +idlelib.tooltip +idlelib.tree +idlelib.undo +idlelib.window +idlelib.zoomheight +idlelib.zzdummy +imaplib +imghdr +_imp +imp +importlib +importlib.abc +importlib._bootstrap +importlib._bootstrap_external +importlib._common +importlib.machinery +importlib.metadata +importlib.resources +importlib.util +inspect +_io +io +ipaddress +itertools +_json +json +json.decoder +json.encoder +json.scanner +json.tool +keyword +lib2to3 +lib2to3.btm_matcher +lib2to3.btm_utils +lib2to3.fixer_base +lib2to3.fixer_util +lib2to3.fixes +lib2to3.fixes.fix_apply +lib2to3.fixes.fix_asserts +lib2to3.fixes.fix_basestring +lib2to3.fixes.fix_buffer +lib2to3.fixes.fix_dict +lib2to3.fixes.fix_except +lib2to3.fixes.fix_exec +lib2to3.fixes.fix_execfile +lib2to3.fixes.fix_exitfunc +lib2to3.fixes.fix_filter +lib2to3.fixes.fix_funcattrs +lib2to3.fixes.fix_future +lib2to3.fixes.fix_getcwdu +lib2to3.fixes.fix_has_key +lib2to3.fixes.fix_idioms +lib2to3.fixes.fix_import +lib2to3.fixes.fix_imports +lib2to3.fixes.fix_imports2 +lib2to3.fixes.fix_input +lib2to3.fixes.fix_intern +lib2to3.fixes.fix_isinstance +lib2to3.fixes.fix_itertools +lib2to3.fixes.fix_itertools_imports +lib2to3.fixes.fix_long +lib2to3.fixes.fix_map +lib2to3.fixes.fix_metaclass +lib2to3.fixes.fix_methodattrs +lib2to3.fixes.fix_ne +lib2to3.fixes.fix_next +lib2to3.fixes.fix_nonzero +lib2to3.fixes.fix_numliterals +lib2to3.fixes.fix_operator +lib2to3.fixes.fix_paren +lib2to3.fixes.fix_print +lib2to3.fixes.fix_raise +lib2to3.fixes.fix_raw_input +lib2to3.fixes.fix_reduce +lib2to3.fixes.fix_reload +lib2to3.fixes.fix_renames +lib2to3.fixes.fix_repr +lib2to3.fixes.fix_set_literal +lib2to3.fixes.fix_standarderror +lib2to3.fixes.fix_sys_exc +lib2to3.fixes.fix_throw +lib2to3.fixes.fix_tuple_params +lib2to3.fixes.fix_types +lib2to3.fixes.fix_unicode +lib2to3.fixes.fix_urllib +lib2to3.fixes.fix_ws_comma +lib2to3.fixes.fix_xrange +lib2to3.fixes.fix_xreadlines +lib2to3.fixes.fix_zip +lib2to3.main +lib2to3.__main__ +lib2to3.patcomp +lib2to3.pgen2 +lib2to3.pgen2.conv +lib2to3.pgen2.driver +lib2to3.pgen2.grammar +lib2to3.pgen2.literals +lib2to3.pgen2.parse +lib2to3.pgen2.pgen +lib2to3.pgen2.token +lib2to3.pgen2.tokenize +lib2to3.pygram +lib2to3.pytree +lib2to3.refactor +lib2to3.tests +lib2to3.tests.data.bom +lib2to3.tests.data.crlf +lib2to3.tests.data.different_encoding +lib2to3.tests.data.false_encoding +lib2to3.tests.data.fixers.bad_order +lib2to3.tests.data.fixers.myfixes +lib2to3.tests.data.fixers.myfixes.fix_explicit +lib2to3.tests.data.fixers.myfixes.fix_first +lib2to3.tests.data.fixers.myfixes.fix_last +lib2to3.tests.data.fixers.myfixes.fix_parrot +lib2to3.tests.data.fixers.myfixes.fix_preorder +lib2to3.tests.data.fixers.no_fixer_cls +lib2to3.tests.data.fixers.parrot_example +lib2to3.tests.data.infinite_recursion +lib2to3.tests.data.py2_test_grammar +lib2to3.tests.data.py3_test_grammar +lib2to3.tests.__main__ +lib2to3.tests.pytree_idempotency +lib2to3.tests.support +lib2to3.tests.test_all_fixers +lib2to3.tests.test_fixers +lib2to3.tests.test_main +lib2to3.tests.test_parser +lib2to3.tests.test_pytree +lib2to3.tests.test_refactor +lib2to3.tests.test_util +lib.libpython3 +linecache +_locale +locale +logging +logging.config +logging.handlers +_lsprof +_lzma +lzma +mailbox +mailcap +__main__ +_markupbase +marshal +math +_md5 +mimetypes +mmap +modulefinder +msilib +msvcrt +_multibytecodec +_multiprocessing +multiprocessing +multiprocessing.connection +multiprocessing.context +multiprocessing.dummy +multiprocessing.dummy.connection +multiprocessing.forkserver +multiprocessing.heap +multiprocessing.managers +multiprocessing.pool +multiprocessing.popen_fork +multiprocessing.popen_forkserver +multiprocessing.popen_spawn_posix +multiprocessing.popen_spawn_win32 +multiprocessing.process +multiprocessing.queues +multiprocessing.reduction +multiprocessing.resource_sharer +multiprocessing.resource_tracker +multiprocessing.sharedctypes +multiprocessing.shared_memory +multiprocessing.spawn +multiprocessing.synchronize +multiprocessing.util +netrc +nis +nntplib +ntpath +nturl2path +numbers +_opcode +opcode +_operator +operator +optparse +os +os.path +ossaudiodev +_osx_support +parser +pathlib +pdb +__phello__.foo +_pickle +pickle +pickletools +pipes +pkgutil +platform +plistlib +poplib +posix +posixpath +_posixshmem +_posixsubprocess +pprint +profile +pstats +pty +pwd +_py_abc +pyclbr +py_compile +_pydecimal +pydoc +pydoc_data +pydoc_data.topics +pyexpat +_pyio +_queue +queue +quopri +_random +random +re +readline +reprlib +resource +rlcompleter +runpy +sched +secrets +select +selectors +_sha1 +_sha256 +_sha3 +_sha512 +shelve +shlex +shutil +_signal +signal +site +_sitebuiltins +smtpd +smtplib +sndhdr +_socket +socket +socketserver +spwd +_sqlite3 +sqlite3 +sqlite3.dbapi2 +sqlite3.dump +sqlite3.test +sqlite3.test.backup +sqlite3.test.dbapi +sqlite3.test.dump +sqlite3.test.factory +sqlite3.test.hooks +sqlite3.test.regression +sqlite3.test.transactions +sqlite3.test.types +sqlite3.test.userfunctions +_sre +sre_compile +sre_constants +sre_parse +_ssl +ssl +_stat +stat +_statistics +statistics +_string +string +stringprep +_strptime +_struct +struct +subprocess +sunau +symbol +_symtable +symtable +sys +sysconfig +_sysconfigdata_x86_64_conda_cos6_linux_gnu +_sysconfigdata_x86_64_conda_linux_gnu +syslog +tabnanny +tarfile +telnetlib +tempfile +termios +test +test.ann_module +test.ann_module2 +test.ann_module3 +test.audiotests +test.autotest +test.bad_coding +test.bad_coding2 +test.bad_getattr +test.bad_getattr2 +test.bad_getattr3 +test.badsyntax_3131 +test.badsyntax_future10 +test.badsyntax_future3 +test.badsyntax_future4 +test.badsyntax_future5 +test.badsyntax_future6 +test.badsyntax_future7 +test.badsyntax_future8 +test.badsyntax_future9 +test.badsyntax_pep3120 +test.bisect_cmd +_testbuffer +test.bytecode_helper +_testcapi +test.coding20731 +test.curses_tests +test.dataclass_module_1 +test.dataclass_module_1_str +test.dataclass_module_2 +test.dataclass_module_2_str +test.datetimetester +test.dis_module +test.doctest_aliases +test.double_const +test.dtracedata.call_stack +test.dtracedata.gc +test.dtracedata.instance +test.dtracedata.line +test.eintrdata.eintr_tester +test.encoded_modules +test.encoded_modules.module_iso_8859_1 +test.encoded_modules.module_koi8_r +test.final_a +test.final_b +test.fork_wait +test.future_test1 +test.future_test2 +test.gdb_sample +test.good_getattr +test.imp_dummy +_testimportmultiple +test.inspect_fodder +test.inspect_fodder2 +_testinternalcapi +test.libregrtest +test.libregrtest.cmdline +test.libregrtest.main +test.libregrtest.pgo +test.libregrtest.refleak +test.libregrtest.runtest +test.libregrtest.runtest_mp +test.libregrtest.save_env +test.libregrtest.setup +test.libregrtest.utils +test.libregrtest.win_utils +test.list_tests +test.lock_tests +test.__main__ +test.make_ssl_certs +test.mapping_tests +test.memory_watchdog +test.mock_socket +test.mod_generics_cache +test.mp_fork_bomb +test.mp_preload +test.multibytecodec_support +_testmultiphase +test.outstanding_bugs +test.pickletester +test.profilee +test.pyclbr_input +test.pydocfodder +test.pydoc_mod +test.pythoninfo +test.regrtest +test.relimport +test.reperf +test.re_tests +test.sample_doctest +test.sample_doctest_no_docstrings +test.sample_doctest_no_doctests +test.seq_tests +test.signalinterproctester +test.sortperf +test.ssl_servers +test.ssltests +test.string_tests +test.subprocessdata.fd_status +test.subprocessdata.input_reader +test.subprocessdata.qcat +test.subprocessdata.qgrep +test.subprocessdata.sigchild_ignore +test.support +test.support.bytecode_helper +test.support.hashlib_helper +test.support.logging_helper +test.support.script_helper +test.support.socket_helper +test.support.testresult +test.test_abc +test.test_abstract_numbers +test.test_aifc +test.test___all__ +test.test_argparse +test.test_array +test.test_asdl_parser +test.test_ast +test.test_asyncgen +test.test_asynchat +test.test_asyncio +test.test_asyncio.echo +test.test_asyncio.echo2 +test.test_asyncio.echo3 +test.test_asyncio.functional +test.test_asyncio.__main__ +test.test_asyncio.test_base_events +test.test_asyncio.test_buffered_proto +test.test_asyncio.test_context +test.test_asyncio.test_events +test.test_asyncio.test_futures +test.test_asyncio.test_locks +test.test_asyncio.test_pep492 +test.test_asyncio.test_proactor_events +test.test_asyncio.test_protocols +test.test_asyncio.test_queues +test.test_asyncio.test_runners +test.test_asyncio.test_selector_events +test.test_asyncio.test_sendfile +test.test_asyncio.test_server +test.test_asyncio.test_sock_lowlevel +test.test_asyncio.test_sslproto +test.test_asyncio.test_streams +test.test_asyncio.test_subprocess +test.test_asyncio.test_tasks +test.test_asyncio.test_transports +test.test_asyncio.test_unix_events +test.test_asyncio.test_windows_events +test.test_asyncio.test_windows_utils +test.test_asyncio.utils +test.test_asyncore +test.test_atexit +test.test_audioop +test.test_audit +test.test_augassign +test.test_base64 +test.test_baseexception +test.test_bdb +test.test_bigaddrspace +test.test_bigmem +test.test_binascii +test.test_binhex +test.test_binop +test.test_bisect +test.test_bool +test.test_buffer +test.test_bufio +test.test_builtin +test.test_bytes +test.test_bz2 +test.test_calendar +test.test_call +test.test_capi +test.test_cgi +test.test_cgitb +test.test_charmapcodec +test.test_class +test.test_clinic +test.test_c_locale_coercion +test.test_cmath +test.test_cmd +test.test_cmd_line +test.test_cmd_line_script +test.test_code +test.testcodec +test.test_codeccallbacks +test.test_codecencodings_cn +test.test_codecencodings_hk +test.test_codecencodings_iso2022 +test.test_codecencodings_jp +test.test_codecencodings_kr +test.test_codecencodings_tw +test.test_codecmaps_cn +test.test_codecmaps_hk +test.test_codecmaps_jp +test.test_codecmaps_kr +test.test_codecmaps_tw +test.test_codecs +test.test_code_module +test.test_codeop +test.test_collections +test.test_colorsys +test.test_compare +test.test_compile +test.test_compileall +test.test_complex +test.test_concurrent_futures +test.test_configparser +test.test_contains +test.test_context +test.test_contextlib +test.test_contextlib_async +test.test_copy +test.test_copyreg +test.test_coroutines +test.test_cprofile +test.test_crashers +test.test_crypt +test.test_csv +test.test_ctypes +test.test_curses +test.test_dataclasses +test.test_datetime +test.test_dbm +test.test_dbm_dumb +test.test_dbm_gnu +test.test_dbm_ndbm +test.test_decimal +test.test_decorators +test.test_defaultdict +test.test_deque +test.test_descr +test.test_descrtut +test.test_devpoll +test.test_dict +test.test_dictcomps +test.test_dict_version +test.test_dictviews +test.test_difflib +test.test_dis +test.test_distutils +test.test_doctest +test.test_doctest2 +test.test_docxmlrpc +test.test_dtrace +test.test_dummy_thread +test.test_dummy_threading +test.test_dynamic +test.test_dynamicclassattribute +test.test_eintr +test.test_email +test.test_email.__main__ +test.test_email.test_asian_codecs +test.test_email.test_contentmanager +test.test_email.test_defect_handling +test.test_email.test_email +test.test_email.test__encoded_words +test.test_email.test_generator +test.test_email.test_headerregistry +test.test_email.test__header_value_parser +test.test_email.test_inversion +test.test_email.test_message +test.test_email.test_parser +test.test_email.test_pickleable +test.test_email.test_policy +test.test_email.test_utils +test.test_email.torture_test +test.test_embed +test.test_ensurepip +test.test_enum +test.test_enumerate +test.test_eof +test.test_epoll +test.test_errno +test.test_exception_hierarchy +test.test_exceptions +test.test_exception_variations +test.test_extcall +test.test_faulthandler +test.test_fcntl +test.test_file +test.test_filecmp +test.test_file_eintr +test.test_fileinput +test.test_fileio +test.test_finalization +test.test_float +test.test_flufl +test.test_fnmatch +test.test_fork1 +test.test_format +test.test_fractions +test.test_frame +test.test_frozen +test.test_fstring +test.test_ftplib +test.test_funcattrs +test.test_functools +test.test___future__ +test.test_future +test.test_future3 +test.test_future4 +test.test_future5 +test.test_gc +test.test_gdb +test.test_generators +test.test_generator_stop +test.test_genericclass +test.test_genericpath +test.test_genexps +test.test_getargs2 +test.test_getopt +test.test_getpass +test.test_gettext +test.test_glob +test.test_global +test.test_grammar +test.test_grp +test.test_gzip +test.test_hash +test.test_hashlib +test.test_heapq +test.test_hmac +test.test_html +test.test_htmlparser +test.test_http_cookiejar +test.test_http_cookies +test.test_httplib +test.test_httpservers +test.test_idle +test.test_imaplib +test.test_imghdr +test.test_imp +test.test_import +test.test_import.data.circular_imports.basic +test.test_import.data.circular_imports.basic2 +test.test_import.data.circular_imports.binding +test.test_import.data.circular_imports.binding2 +test.test_import.data.circular_imports.from_cycle1 +test.test_import.data.circular_imports.from_cycle2 +test.test_import.data.circular_imports.indirect +test.test_import.data.circular_imports.rebinding +test.test_import.data.circular_imports.rebinding2 +test.test_import.data.circular_imports.source +test.test_import.data.circular_imports.subpackage +test.test_import.data.circular_imports.subpkg.subpackage2 +test.test_import.data.circular_imports.subpkg.util +test.test_import.data.circular_imports.use +test.test_import.data.circular_imports.util +test.test_import.data.package +test.test_import.data.package2.submodule1 +test.test_import.data.package2.submodule2 +test.test_import.data.package.submodule +test.test_importlib +test.test_importlib.abc +test.test_importlib.builtin +test.test_importlib.builtin.__main__ +test.test_importlib.builtin.test_finder +test.test_importlib.builtin.test_loader +test.test_importlib.data +test.test_importlib.data01 +test.test_importlib.data01.subdirectory +test.test_importlib.data02 +test.test_importlib.data02.one +test.test_importlib.data02.two +test.test_importlib.data03 +test.test_importlib.data03.namespace.portion1 +test.test_importlib.data03.namespace.portion2 +test.test_importlib.extension +test.test_importlib.extension.__main__ +test.test_importlib.extension.test_case_sensitivity +test.test_importlib.extension.test_finder +test.test_importlib.extension.test_loader +test.test_importlib.extension.test_path_hook +test.test_importlib.fixtures +test.test_importlib.frozen +test.test_importlib.frozen.__main__ +test.test_importlib.frozen.test_finder +test.test_importlib.frozen.test_loader +test.test_importlib.import_ +test.test_importlib.import_.__main__ +test.test_importlib.import_.test_api +test.test_importlib.import_.test_caching +test.test_importlib.import_.test_fromlist +test.test_importlib.import_.test___loader__ +test.test_importlib.import_.test_meta_path +test.test_importlib.import_.test___package__ +test.test_importlib.import_.test_packages +test.test_importlib.import_.test_path +test.test_importlib.import_.test_relative_imports +test.test_importlib.__main__ +test.test_importlib.namespace_pkgs.both_portions.foo.one +test.test_importlib.namespace_pkgs.both_portions.foo.two +test.test_importlib.namespace_pkgs.module_and_namespace_package.a_test +test.test_importlib.namespace_pkgs.not_a_namespace_pkg.foo +test.test_importlib.namespace_pkgs.not_a_namespace_pkg.foo.one +test.test_importlib.namespace_pkgs.portion1.foo.one +test.test_importlib.namespace_pkgs.portion2.foo.two +test.test_importlib.namespace_pkgs.project1.parent.child.one +test.test_importlib.namespace_pkgs.project2.parent.child.two +test.test_importlib.namespace_pkgs.project3.parent.child.three +test.test_importlib.source +test.test_importlib.source.__main__ +test.test_importlib.source.test_case_sensitivity +test.test_importlib.source.test_file_loader +test.test_importlib.source.test_finder +test.test_importlib.source.test_path_hook +test.test_importlib.source.test_source_encoding +test.test_importlib.test_abc +test.test_importlib.test_api +test.test_importlib.test_lazy +test.test_importlib.test_locks +test.test_importlib.test_main +test.test_importlib.test_metadata_api +test.test_importlib.test_namespace_pkgs +test.test_importlib.test_open +test.test_importlib.test_path +test.test_importlib.test_read +test.test_importlib.test_resource +test.test_importlib.test_spec +test.test_importlib.test_util +test.test_importlib.test_windows +test.test_importlib.test_zip +test.test_importlib.util +test.test_importlib.zipdata01 +test.test_importlib.zipdata02 +test.test_import.__main__ +test.test_index +test.test_inspect +test.test_int +test.test_int_literal +test.test_io +test.test_ioctl +test.test_ipaddress +test.test_isinstance +test.test_iter +test.test_iterlen +test.test_itertools +test.test_json +test.test_json.__main__ +test.test_json.test_decode +test.test_json.test_default +test.test_json.test_dump +test.test_json.test_encode_basestring_ascii +test.test_json.test_enum +test.test_json.test_fail +test.test_json.test_float +test.test_json.test_indent +test.test_json.test_pass1 +test.test_json.test_pass2 +test.test_json.test_pass3 +test.test_json.test_recursion +test.test_json.test_scanstring +test.test_json.test_separators +test.test_json.test_speedups +test.test_json.test_tool +test.test_json.test_unicode +test.test_keyword +test.test_keywordonlyarg +test.test_kqueue +test.test_largefile +test.test_lib2to3 +test.test_linecache +test.test_list +test.test_listcomps +test.test_lltrace +test.test__locale +test.test_locale +test.test_logging +test.test_long +test.test_longexp +test.test_lzma +test.test_mailbox +test.test_mailcap +test.test_marshal +test.test_math +test.test_memoryio +test.test_memoryview +test.test_metaclass +test.test_mimetypes +test.test_minidom +test.test_mmap +test.test_module +test.test_modulefinder +test.test_msilib +test.test_multibytecodec +test._test_multiprocessing +test.test_multiprocessing_fork +test.test_multiprocessing_forkserver +test.test_multiprocessing_main_handling +test.test_multiprocessing_spawn +test.test_named_expressions +test.test_netrc +test.test_nis +test.test_nntplib +test.test_normalization +test.test_ntpath +test.test_numeric_tower +test.test__opcode +test.test_opcodes +test.test_openpty +test.test_operator +test.test_optparse +test.test_ordered_dict +test.test_os +test.test_ossaudiodev +test.test_osx_env +test.test__osx_support +test.test_parser +test.test_pathlib +test.test_pdb +test.test_peepholer +test.test_pickle +test.test_picklebuffer +test.test_pickletools +test.test_pipes +test.test_pkg +test.test_pkgimport +test.test_pkgutil +test.test_platform +test.test_plistlib +test.test_poll +test.test_popen +test.test_poplib +test.test_positional_only_arg +test.test_posix +test.test_posixpath +test.test_pow +test.test_pprint +test.test_print +test.test_profile +test.test_property +test.test_pstats +test.test_pty +test.test_pulldom +test.test_pwd +test.test_pyclbr +test.test_py_compile +test.test_pydoc +test.test_pyexpat +test.test_queue +test.test_quopri +test.test_raise +test.test_random +test.test_range +test.test_re +test.test_readline +test.test_regrtest +test.test_repl +test.test_reprlib +test.test_resource +test.test_richcmp +test.test_rlcompleter +test.test_robotparser +test.test_runpy +test.test_sax +test.test_sched +test.test_scope +test.test_script_helper +test.test_secrets +test.test_select +test.test_selectors +test.test_set +test.test_setcomps +test.test_shelve +test.test_shlex +test.test_shutil +test.test_signal +test.test_site +test.test_slice +test.test_smtpd +test.test_smtplib +test.test_smtpnet +test.test_sndhdr +test.test_socket +test.test_socketserver +test.test_sort +test.test_source_encoding +test.test_spwd +test.test_sqlite +test.test_ssl +test.test_startfile +test.test_stat +test.test_statistics +test.test_strftime +test.test_string +test.test_string_literals +test.test_stringprep +test.test_strptime +test.test_strtod +test.test_struct +test.test_structmembers +test.test_structseq +test.test_subclassinit +test.test_subprocess +test.test_sunau +test.test_sundry +test.test_super +test.test_support +test.test_symbol +test.test_symtable +test.test_syntax +test.test_sys +test.test_sysconfig +test.test_syslog +test.test_sys_setprofile +test.test_sys_settrace +test.test_tabnanny +test.test_tarfile +test.test_tcl +test.test_telnetlib +test.test_tempfile +test.test_textwrap +test.test_thread +test.test_threaded_import +test.test_threadedtempfile +test.test_threading +test.test_threading_local +test.test_threadsignals +test.test_time +test.test_timeit +test.test_timeout +test.test_tix +test.test_tk +test.test_tokenize +test.test_tools +test.test_tools.__main__ +test.test_tools.test_fixcid +test.test_tools.test_gprof2html +test.test_tools.test_i18n +test.test_tools.test_lll +test.test_tools.test_md5sum +test.test_tools.test_pathfix +test.test_tools.test_pdeps +test.test_tools.test_pindent +test.test_tools.test_reindent +test.test_tools.test_sundry +test.test_tools.test_unparse +test.test_trace +test.test_traceback +test.test_tracemalloc +test.test_ttk_guionly +test.test_ttk_textonly +test.test_tuple +test.test_turtle +test.test_typechecks +test.test_type_comments +test.test_types +test.test_typing +test.test_ucn +test.test_unary +test.test_unicode +test.test_unicodedata +test.test_unicode_file +test.test_unicode_file_functions +test.test_unicode_identifiers +test.test_unittest +test.test_univnewlines +test.test_unpack +test.test_unpack_ex +test.test_urllib +test.test_urllib2 +test.test_urllib2_localnet +test.test_urllib2net +test.test_urllibnet +test.test_urllib_response +test.test_urlparse +test.test_userdict +test.test_userlist +test.test_userstring +test.test_utf8_mode +test.test_utf8source +test.test_uu +test.test_uuid +test.test_venv +test.test_wait3 +test.test_wait4 +test.test_warnings +test.test_warnings.data.import_warning +test.test_warnings.data.stacklevel +test.test_warnings.__main__ +test.test_wave +test.test_weakref +test.test_weakset +test.test_webbrowser +test.test_winconsoleio +test.test_winreg +test.test_winsound +test.test_with +test.test_wsgiref +test.test_xdrlib +test.test_xml_dom_minicompat +test.test_xml_etree +test.test_xml_etree_c +test.test_xmlrpc +test.test_xmlrpc_net +test.test__xxsubinterpreters +test.test_xxtestfuzz +test.test_yield_from +test.test_zipapp +test.test_zipfile +test.test_zipfile64 +test.test_zipimport +test.test_zipimport_support +test.test_zlib +test.tf_inherit_check +test.threaded_import_hangers +test.time_hashlib +test.tracedmodules +test.tracedmodules.testmod +test.win_console_handler +test.xmltests +test.ziptestdata.testdata_module_inside_zip +textwrap +this +_thread +threading +_threading_local +time +timeit +_tkinter +tkinter +tkinter.colorchooser +tkinter.commondialog +tkinter.constants +tkinter.dialog +tkinter.dnd +tkinter.filedialog +tkinter.font +tkinter.__main__ +tkinter.messagebox +tkinter.scrolledtext +tkinter.simpledialog +tkinter.test +tkinter.test.runtktests +tkinter.test.support +tkinter.test.test_tkinter +tkinter.test.test_tkinter.test_font +tkinter.test.test_tkinter.test_geometry_managers +tkinter.test.test_tkinter.test_images +tkinter.test.test_tkinter.test_loadtk +tkinter.test.test_tkinter.test_misc +tkinter.test.test_tkinter.test_text +tkinter.test.test_tkinter.test_variables +tkinter.test.test_tkinter.test_widgets +tkinter.test.test_ttk +tkinter.test.test_ttk.test_extensions +tkinter.test.test_ttk.test_functions +tkinter.test.test_ttk.test_style +tkinter.test.test_ttk.test_widgets +tkinter.test.widget_tests +tkinter.tix +tkinter.ttk +token +tokenize +trace +traceback +_tracemalloc +tracemalloc +tty +turtle +turtledemo +turtledemo.bytedesign +turtledemo.chaos +turtledemo.clock +turtledemo.colormixer +turtledemo.forest +turtledemo.fractalcurves +turtledemo.lindenmayer +turtledemo.__main__ +turtledemo.minimal_hanoi +turtledemo.nim +turtledemo.paint +turtledemo.peace +turtledemo.penrose +turtledemo.planet_and_moon +turtledemo.rosette +turtledemo.round_dance +turtledemo.sorting_animate +turtledemo.tree +turtledemo.two_canvases +turtledemo.yinyang +types +typing +typing.io +typing.re +unicodedata +unittest +unittest.async_case +unittest.case +unittest.loader +unittest._log +unittest.__main__ +unittest.main +unittest.mock +unittest.result +unittest.runner +unittest.signals +unittest.suite +unittest.test +unittest.test.dummy +unittest.test.__main__ +unittest.test.support +unittest.test.test_assertions +unittest.test.test_async_case +unittest.test.test_break +unittest.test.test_case +unittest.test.test_discovery +unittest.test.test_functiontestcase +unittest.test.test_loader +unittest.test.testmock +unittest.test.testmock.__main__ +unittest.test.testmock.support +unittest.test.testmock.testasync +unittest.test.testmock.testcallable +unittest.test.testmock.testhelpers +unittest.test.testmock.testmagicmethods +unittest.test.testmock.testmock +unittest.test.testmock.testpatch +unittest.test.testmock.testsealable +unittest.test.testmock.testsentinel +unittest.test.testmock.testwith +unittest.test.test_program +unittest.test.test_result +unittest.test.test_runner +unittest.test.test_setups +unittest.test.test_skipping +unittest.test.test_suite +unittest.test._test_warnings +unittest.util +urllib +urllib.error +urllib.parse +urllib.request +urllib.response +urllib.robotparser +uu +_uuid +uuid +venv +venv.__main__ +_warnings +warnings +wave +_weakref +weakref +_weakrefset +webbrowser +winreg +winsound +wsgiref +wsgiref.handlers +wsgiref.headers +wsgiref.simple_server +wsgiref.util +wsgiref.validate +xdrlib +xml +xml.dom +xml.dom.domreg +xml.dom.expatbuilder +xml.dom.minicompat +xml.dom.minidom +xml.dom.NodeFilter +xml.dom.pulldom +xml.dom.xmlbuilder +xml.etree +xml.etree.cElementTree +xml.etree.ElementInclude +xml.etree.ElementPath +xml.etree.ElementTree +xml.parsers +xml.parsers.expat +xml.parsers.expat.errors +xml.parsers.expat.model +xmlrpc +xmlrpc.client +xmlrpc.server +xml.sax +xml.sax._exceptions +xml.sax.expatreader +xml.sax.handler +xml.sax.saxutils +xml.sax.xmlreader +xxlimited +_xxsubinterpreters +xxsubtype +_xxtestfuzz +zipapp +zipfile +zipimport +zlib +zoneinfo +zoneinfo._common +zoneinfo._tzpath +zoneinfo._zoneinfo diff --git a/goex/exec_engine/utils.py b/goex/exec_engine/utils.py new file mode 100644 index 000000000..fa1a7bb8d --- /dev/null +++ b/goex/exec_engine/utils.py @@ -0,0 +1,26 @@ +"""Common utility functions and classes""" + +import os +from pathlib import Path +from typing import NewType + +ROOT_FOLDER_PATH = os.path.dirname(Path(os.path.realpath(__file__)).parent) +DEFAULT_CHECKPOINT_PATH = os.path.join(ROOT_FOLDER_PATH, 'checkpoints') + +SQL_Type = NewType("SQL_Type", str) +Filesystem_Type = NewType("Filesystem_Type", str) +RESTful_Type = NewType("RESTful_Type", str) + +def format_container_logs(container): + docker_out = [] + for log in container.logs(stdout=True, stderr=False, stream=True): + log = log.decode("utf-8") + if log == "\n": + continue + else: + if log[-1] == "\n": + log = log[:-1] + docker_out.append(log) + + docker_debug = container.logs(stdout=False, stderr=True).decode("utf-8") + return docker_out, docker_debug diff --git a/goex/function/dummy_key.json b/goex/function/dummy_key.json new file mode 100644 index 000000000..1b5429a0e --- /dev/null +++ b/goex/function/dummy_key.json @@ -0,0 +1 @@ +{"channel_id": "D04HS7LEX17"} \ No newline at end of file diff --git a/goex/function/function.json b/goex/function/function.json new file mode 100644 index 000000000..73e9ec338 --- /dev/null +++ b/goex/function/function.json @@ -0,0 +1,190 @@ +[ + { + "type": "function", + "function": { + "name": "slack_send_message", + "description": "Send an slack message to a channel.", + "parameters": { + "type": "object", + "properties": { + "channel_id": { + "type": "string", + "description": "The channel id to send the message to. Example: 'D04HS7LEX17'." + }, + "text": { + "type": "string", + "description": "The text of the message. Example: 'hello bob'." + } + }, + "required": ["channel_id", "text"] + } + } + }, + { + "type": "function", + "function": { + "name": "slack_send_message_dummy", + "description": "Send an slack message to a channel without specifying channel id.", + "parameters": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The text of the message. Example: 'hello bob'." + } + }, + "required": ["text"] + } + } + }, + { + "type": "function", + "function": { + "name": "slack_delete_most_recent_message", + "description": "Delete the latest message you sent in a channel.", + "parameters": { + "type": "object", + "properties": { + "channel_id": { + "type": "string", + "description": "The channel id to delete the message from. Example: 'D04HS7LEX17'." + } + }, + "required": ["channel_id"] + } + } + }, + { + "type": "function", + "function": { + "name": "slack_delete_most_recent_message_dummy", + "description": "Delete the latest message you sent in a channel without specifying channel id.", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } + }, + { + "type": "function", + "function": { + "name": "slack_list_channels", + "description": "List all the channels in your slack workspace.", + "parameters": { + "type": "object", + "properties": {}, + "required": [] + } + } + }, + { + "type": "function", + "function": { + "name": "slack_add_emoji_reaction", + "description": "Add an emoji reaction to a message in the channel.", + "parameters": { + "type": "object", + "properties": { + "channel_id": { + "type": "string", + "description": "The id of the channel to add emoji reaction to. Example: 'D04HS7LEX17'." + }, + "emoji_name": { + "type": "string", + "description": "The name of the emoji to add. Example: 'thumbsup'." + }, + "timestamp_id": { + "type": "string", + "description": "The timestamp id of the message to add emoji reaction to. Example: '1712451307.299279'." + } + }, + "required": ["channel_id", "emoji_name", "timestamp_id"] + } + } + }, + { + "type": "function", + "function": { + "name": "slack_remove_latest_emoji_reaction", + "description": "Remove an emoji reaction to the latest message in the channel.", + "parameters": { + "type": "object", + "properties": { + "channel_id": { + "type": "string", + "description": "The channel id to remove emoji reaction from. Example: 'D04HS7LEX17'." + }, + "emoji_name": { + "type": "string", + "description": "The name of the emoji to add. Example: 'thumbsup'." + } + }, + "required": ["channel_id", "emoji_name"] + } + } + }, + { + "type": "function", + "function": { + "name": "slack_upload_file", + "description": "Upload a file to a channel.", + "parameters": { + "type": "object", + "properties": { + "channel_id": { + "type": "string", + "description": "The channel id to delete the message from. Example: 'D04HS7LEX17'." + }, + "initial_comment": { + "type": "string", + "description": "Initial text comments we add to the file. Example: 'Personal file'." + }, + "file_name": { + "type": "string", + "description": "The name of the file to be uploaded Example: 'README.md'." + } + }, + "required": ["channel_id", "initial_comment", "file_name"] + } + } + }, + { + "type": "function", + "function": { + "name": "slack_delete_latest_file", + "description": "Delete the latest file and its corresponding message in a channel.", + "parameters": { + "type": "object", + "properties": { + "channel_id": { + "type": "string", + "description": "The channel id to delete the file from. Example: 'D04HS7LEX17'." + } + }, + "required": ["channel_id"] + } + } + }, + { + "type": "function", + "function": { + "name": "slack_read_messages", + "description": "Read the most recent k messages in a channel.", + "parameters": { + "type": "object", + "properties": { + "channel_id": { + "type": "string", + "description": "The channel id to read messages from. Example: 'D04HS7LEX17'." + }, + "limit": { + "type": "integer", + "description": "Number of messages to be read from the channel. Default: 10. Example: 1." + } + }, + "required": ["channel_id"] + } + } + } +] diff --git a/goex/function/slack_add_emoji_reaction.py b/goex/function/slack_add_emoji_reaction.py new file mode 100644 index 000000000..1657ee1e2 --- /dev/null +++ b/goex/function/slack_add_emoji_reaction.py @@ -0,0 +1,30 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + + +def slack_add_emoji_reaction(channel_id, emoji_name, timestamp_id): + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + try: + # Call the chat.postMessage method using the WebClient + result = client.reactions_add( + # The name of the conversation + channel=channel_id, + name=emoji_name, + timestamp=timestamp_id, + ) + print("Message sent successfully.", result) + + except SlackApiError as e: + # Print the error + print(f"Error sending message: {e.response['error']}") + raise diff --git a/goex/function/slack_delete_latest_file.py b/goex/function/slack_delete_latest_file.py new file mode 100644 index 000000000..27bce3f0c --- /dev/null +++ b/goex/function/slack_delete_latest_file.py @@ -0,0 +1,44 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + +def slack_delete_latest_file(channel_id): + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + try: + result = client.conversations_history( + channel=channel_id, + limit=10 + ) + messages = result.get('messages', []) + + file_id = None + msg_ts = None + for message in messages: + bots = message.get('files', []) + if len(bots) > 0: + file_id = bots[0].get('id', []) + msg_ts = message.get('ts', []) + break + if file_id: + # Delete the file + deletion_result = client.files_delete(file=file_id) + print("File deleted successfully.", deletion_result) + + msg_result = client.chat_delete(channel=channel_id, ts=msg_ts) + print("Message deleted successfully.", msg_result) + + else: + print("File not found.") + + except SlackApiError as e: + print(f"Error deleting file: {e.response['error']}") + raise diff --git a/goex/function/slack_delete_most_recent_message.py b/goex/function/slack_delete_most_recent_message.py new file mode 100644 index 000000000..06ca38fc1 --- /dev/null +++ b/goex/function/slack_delete_most_recent_message.py @@ -0,0 +1,37 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + + +def slack_delete_most_recent_message(channel_id): + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + + try: + result = client.conversations_history( + channel=channel_id, + limit=1 + ) + last_message = result['messages'][0]['ts'] if result['messages'] else None + if last_message: + # Call the chat.delete method to delete the most recent message + delete_result = client.chat_delete( + channel=channel_id, + ts=last_message + ) + print("Last message deleted successfully.", delete_result) + else: + print("No message found to delete.") + + except SlackApiError as e: + # Print the error + print(f"Error sending message: {e.response['error']}") + raise diff --git a/goex/function/slack_delete_most_recent_message_dummy.py b/goex/function/slack_delete_most_recent_message_dummy.py new file mode 100644 index 000000000..2cd2aa2f6 --- /dev/null +++ b/goex/function/slack_delete_most_recent_message_dummy.py @@ -0,0 +1,37 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + + +def slack_delete_most_recent_message_dummy(): + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + + try: + result = client.conversations_history( + channel="<>", + limit=1 + ) + last_message = result['messages'][0]['ts'] if result['messages'] else None + if last_message: + # Call the chat.delete method to delete the most recent message + delete_result = client.chat_delete( + channel="<>", + ts=last_message + ) + print("Last message deleted successfully.", delete_result) + else: + print("No message found to delete.") + + except SlackApiError as e: + # Print the error + print(f"Error sending message: {e.response['error']}") + raise diff --git a/goex/function/slack_list_channels.py b/goex/function/slack_list_channels.py new file mode 100644 index 000000000..c72f31238 --- /dev/null +++ b/goex/function/slack_list_channels.py @@ -0,0 +1,27 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + + +def list_channels(): + # channel is a string start with @ + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + try: + # Call the conversations.list method using the WebClient + result = client.conversations_list() + print(result["channels"]) + + except SlackApiError as e: + # Print the error + print(f"Error sending message: {e.response['error']}") + raise + diff --git a/goex/function/slack_read_messages.py b/goex/function/slack_read_messages.py new file mode 100644 index 000000000..ec7287da3 --- /dev/null +++ b/goex/function/slack_read_messages.py @@ -0,0 +1,32 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + + +def slack_read_messages(channel_id, limit=100): + # channel is a string start with @ + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + try: + # Call the conversations.history method using the WebClient + # conversations.history returns the first 100 messages by default + # These results are paginated, see: https://api.slack.com/methods/conversations.history$pagination + result = client.conversations_history(channel=channel_id, limit=limit) + + conversation_history = result["messages"] + + # Print results + print("Message read successfully.", conversation_history, channel_id) + + except SlackApiError as e: + # Print the error + print(f"Error sending message: {e.response['error']}") + raise \ No newline at end of file diff --git a/goex/function/slack_remove_latest_emoji_reaction.py b/goex/function/slack_remove_latest_emoji_reaction.py new file mode 100644 index 000000000..ca311a55a --- /dev/null +++ b/goex/function/slack_remove_latest_emoji_reaction.py @@ -0,0 +1,38 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + + +def slack_remove_latest_emoji_reaction(channel_name, emoji_name): + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + try: + # Call the chat.postMessage method using the WebClient + result = client.conversations_history( + channel=channel_name, + limit=1 + ) + last_message = result['messages'][0]['ts'] if result['messages'] else None + if last_message: + react_result = client.reactions_remove( + # The name of the conversation + channel=channel_name, + name=emoji_name, + timestamp=last_message + ) + print("Message sent successfully.", react_result) + else: + print("No message found to remove react.") + + except SlackApiError as e: + # Print the error + print(f"Error sending message: {e.response['error']}") + raise diff --git a/goex/function/slack_send_message.py b/goex/function/slack_send_message.py new file mode 100644 index 000000000..1be62c05e --- /dev/null +++ b/goex/function/slack_send_message.py @@ -0,0 +1,29 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + + +def slack_send_message(channel_id, text): + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + + try: + # Call the chat.postMessage method using the WebClient + result = client.chat_postMessage( + channel=channel_id, + text=text + ) + print("Message sent successfully.", result) + + except SlackApiError as e: + # Print the error + print(f"Error sending message: {e.response['error']}") + raise diff --git a/goex/function/slack_send_message_dummy.py b/goex/function/slack_send_message_dummy.py new file mode 100644 index 000000000..025517597 --- /dev/null +++ b/goex/function/slack_send_message_dummy.py @@ -0,0 +1,29 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + + +def slack_send_message_dummy(text): + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + + try: + # Call the chat.postMessage method using the WebClient + result = client.chat_postMessage( + channel="<>", + text=text + ) + print("Message sent successfully.", result) + + except SlackApiError as e: + # Print the error + print(f"Error sending message: {e.response['error']}") + raise diff --git a/goex/function/slack_upload_file.py b/goex/function/slack_upload_file.py new file mode 100644 index 000000000..b2035c6ca --- /dev/null +++ b/goex/function/slack_upload_file.py @@ -0,0 +1,29 @@ +import os +import pickle +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + + +def slack_upload_file(channel_id, initial_comment, file_name): + # Load Slack credentials + credentials_path = './credentials/slack/token.pickle' + if os.path.exists(credentials_path): + with open(credentials_path, 'rb') as token_file: + slack_token = pickle.load(token_file) + else: + raise FileNotFoundError("Slack token file not found.") + client = WebClient(token=slack_token) + + try: + # Call the chat.postMessage method using the WebClient + result = client.files_upload_v2( + channels=channel_id, + initial_comment=initial_comment, + file=file_name, + ) + print("Message sent successfully.", result) + + except SlackApiError as e: + # Print the error + print(f"Error sending message: {e.response['error']}") + raise diff --git a/goex/main.py b/goex/main.py new file mode 100644 index 000000000..81673fd2d --- /dev/null +++ b/goex/main.py @@ -0,0 +1,278 @@ +"""The driver file for the program.""" + +import re +import os +from typing import Union +from collections import deque +from pathlib import Path + +from exec_engine.db_manager import DBManager +from exec_engine.api_executor import APIExecutor, PythonAPIExecutor +from exec_engine.docker_sandbox import DockerSandbox +from exec_engine.fs_manager import FSManager +from exec_engine.utils import SQL_Type, RESTful_Type, Filesystem_Type + +from exec_engine.pipeline import generate_command + +ROOT_FOLDER_PATH = os.path.dirname(Path(os.path.realpath(__file__))) +CHECKPOINTS_FOLDER_PATH = os.path.join(ROOT_FOLDER_PATH, "docker/mysql_docker/checkpoints") + +class ExecutionEngine: + """Can identify and execute API calls and their reversals.""" + + def __init__(self, history_length: int = 10, path=None, generate_mode='default'): + # Keys for LLMs + self.openai_api_key = None + self.anthropic_api_key = None + # Initialize the instances of the classes + self.api_executor = APIExecutor() + self.docker_sandbox = DockerSandbox() + + self.fs_initialized = False + self.db_initialized = False + self.path = path + self.generate_mode = generate_mode + + # Dry Run is option 2, no dry run is option 1, user can specify which types to dry-run + self.dry_run_dict = { + SQL_Type: False, + Filesystem_Type: False, + RESTful_Type: False, + } + + # Initialize the API and ~API pair circular queue + self.api_history_pair_queue_dict = { + SQL_Type: deque(maxlen=history_length), + Filesystem_Type: deque(maxlen=history_length), + RESTful_Type: deque(maxlen=history_length), + } + + def gen_api_pair(self, input_prompt: str, api_type: str, credentials, model) -> list: + """Generate an API call and its reversal""" + if api_type == RESTful_Type: + forward_call = generate_command(input_prompt, credentials, generate_mode=self.generate_mode, openai_model=model) + backward_call = self.api_executor.try_get_backward_call(forward_call, input_prompt, credentials, api_type, + generate_mode=self.generate_mode, model=model) + return forward_call, backward_call + + elif api_type == SQL_Type: + input_p = self.db_manager.task_to_prompt(input_prompt) + forward_call = generate_command(input_p, credentials, api_type=SQL_Type, generate_mode=self.generate_mode, openai_model=model) + + reverse_p = self.db_manager.task_to_prompt(forward_call, forward=False) + backward_call = generate_command(reverse_p, credentials, api_type=SQL_Type, generate_mode=self.generate_mode, openai_model=model) + return forward_call, backward_call + + elif api_type == Filesystem_Type: + input_p = self.fs_manager.task_to_prompt(input_prompt) + forward_call = generate_command(input_p, credentials, api_type=Filesystem_Type, generate_mode=self.generate_mode, openai_model=model) + + reverse_p = self.fs_manager.task_to_prompt(forward_call, forward=False) + backward_call = generate_command(reverse_p, credentials, api_type=Filesystem_Type, generate_mode=self.generate_mode, openai_model=model) + return forward_call, backward_call + raise NotImplementedError + + + def run_prompt(self, input_prompt: str, api_type: str): + credentials = None # TODO: Work out credentials logic + api_call, neg_api_call = self.gen_api_pair(input_prompt, api_type, credentials, model="gpt-4-turbo-preview") + + exec_result = self.exec_api_call(api_call, api_type, neg_api_call) + + + def test_api_pair_on_docker(self, api_call, neg_api_call, api_type, environment=None) -> bool: + """Tests an API call and its negation by executing them in a sandbox and verifying the state reversion.""" + + if api_type != SQL_Type and api_type != Filesystem_Type: + # 1. Create Docker sandbox + sandbox = self.docker_sandbox.create_sandbox() + # 2. Execute API call and its reversal in the sandbox + self.api_executor.execute_api_call(api_call) + self.api_executor.execute_api_call(neg_api_call) + # 3. Check if the state is reverted successfully + success = self.check_state_reversion() + # 4. Log the result in the database + self.db_manager.log_api_pair(api_call, neg_api_call, success) + # 5. Delete the sandbox + self.docker_sandbox.delete_sandbox(sandbox) + elif api_type == SQL_Type: + + test_prompt = "Table Schemas:\n" + test_prompt += self.db_manager.get_schema_as_string() + test_prompt += f"\nSQL call: {api_call}\n" + test_prompt += f"Reverse call: {neg_api_call}\n\n" + test_prompt += "Given the table schemas, SQL call, and its reversal call, " + \ + "write a Python script to test the SQL call to see if the reversal works. " + \ + "To test, create a new bare minimum database to test this out for " + \ + f"{self.db_manager.db_type}, run the SQL call and its reversal, and check " + \ + "that the start state and the end state are the same. Print out only True " + \ + "if it works, otherwise print out False and print the table contents to stderr. Don't print anything else." + + if self.db_manager.TEST_CONFIG: + test_prompt += f"Make sure to use the config to access the DB: {self.db_manager.TEST_CONFIG}" + + + elif api_type == Filesystem_Type: + test_prompt = ( + "Path: /sandbox/test_dir" + f"Shell Command: {api_call}" + f"Reverse Command: {neg_api_call}" + "Given a directory path, and a shell command and the reverse command of the shell command, " + "can you write a python script to test that the reverse command properly reverses the shell command? " + "Duplicate the given directory, and perform execution in the duplicate directory only. " + "Print out only True if it works, otherwise print out False and print errors to stderr. Don't print anything else.\n\n" + ) + + + res = generate_command(test_prompt, generate_mode=self.generate_mode) + + # get script + result = self._run_code_in_container(code=res, debug=True, api_type=api_type) + + if not result: + return False + + out, debug = result['output'], result['debug'] + if len(out) >= 1: + revert_success = bool(out[-1]) + return revert_success + else: + print(f"An issue occurred during the dry run: {debug}") + return False + + def exec_api_call(self, api_call: str, api_type, debug_neg: str = None) -> str: + + if debug_neg: + neg_api_call = debug_neg + + + if self.dry_run_dict[api_type] and debug_neg: + revert_success = self.test_api_pair_on_docker(api_call, neg_api_call, api_type) + if not revert_success: + raise RuntimeError("Dry Run Failed") + + if api_type == RESTful_Type: + self._exec_restful_call(api_call) + + payload = (api_call, neg_api_call) + + else: + payload = (api_call, debug_neg) + if api_type == SQL_Type: + if self.db_manager: + self._exec_db_call(api_call) + else: + payload = None + elif api_type == Filesystem_Type: + self._exec_filesystem_call(api_call) + + if payload: + self._add_api_reverse_to_queue(api_type, payload) + + def undo_api_call(self, api_type: Union[SQL_Type, Filesystem_Type, RESTful_Type], services=None, option=2): + api_call, neg_api_call = self._pop_api_reverse_from_queue(api_type) + + if not api_call: + print('History is empty') + return + + if option == 2: + # Not a RESTful API negation pair, transaction based + self._undo_transaction(api_type) + + else: + if api_type == RESTful_Type: + self.api_executor.execute_api_call() + + + def commit_api_call(self, api_type, arg=None): + if api_type != RESTful_Type: + self._commit_transaction(api_type, message=arg) + else: + raise NotImplementedError + self._reset_api_history_queue(api_type) + + def _undo_transaction(self, api_type): + if api_type == SQL_Type: + self.db_manager.rollback_db_calls() + elif api_type == Filesystem_Type: + self.fs_manager.revert() + else: + raise NotImplementedError + + def _commit_transaction(self, api_type, message=None): + if api_type == SQL_Type: + self.db_manager.commit_db_calls() + elif api_type == Filesystem_Type: + if not message: + message = 'Auto-commit via FSManager' + self.fs_manager.commit(message=message) + + else: + raise NotImplementedError + + def initialize_db(self, debug_manager: DBManager=None): + print('Initialized DB Manager') + self.db_manager = debug_manager + self.db_initialized = True + + def initialize_fs(self, debug_path=None, git_init=True): + print('Initialized FS Manager') + self.fs_manager = FSManager(debug_path, git_init=git_init) + self.fs_manager.initialize_version_control() + self.fs_initialized = True + + def set_dry_run(self, api_type, on): + self.dry_run_dict[api_type] = on + + def _exec_db_call(self, call) -> bool: + """Execute a SQL API call""" + if not self.db_initialized: + self.initialize_db() + + if re.match( + r"^SELECT\b", call, re.IGNORECASE + ): + return self.db_manager.fetch_db_call(call) + else: + self.db_manager.execute_db_call(call) + + def _exec_filesystem_call(self, call) -> bool: + """Execute a file system API call""" + if not self.fs_initialized: + self.initialize_fs() + self.fs_manager.initialize_version_control() # make sure that git exists on every call + self.fs_manager.execute(call, display=True) + + def _exec_restful_call(self, call) -> bool: + """Execute a RESTful API call""" + raise NotImplementedError + + def _add_api_reverse_to_queue(self, api_type: Union[SQL_Type, Filesystem_Type, RESTful_Type], payload): + self.api_history_pair_queue_dict[api_type].append(payload) + + def _pop_api_reverse_from_queue(self, api_type: Union[SQL_Type, Filesystem_Type, RESTful_Type]): + """Remove a pair of API calls from the list of API pairs""" + # Dequeue in deque + return self.api_history_pair_queue_dict[api_type].pop() if self.api_history_pair_queue_dict[api_type] else None + + def _reset_api_history_queue(self, api_type: Union[SQL_Type, Filesystem_Type, RESTful_Type]): + """Reset the API history queue""" + # Clear the deque + self.api_history_pair_queue_dict[api_type].clear() + + def _run_code_in_container(self, code, debug=False, api_type=RESTful_Type): + try: + volume_path = None + if api_type == Filesystem_Type: + volume_path = os.path.abspath(self.fs_manager.fs_path) + image = self.docker_sandbox.create_image_from_code(code, api_type) + response = self.docker_sandbox.create_python_sandbox(code, image, attached_volume=volume_path) + + if not debug: + return response['output'] + else: + return response + except Exception as e: + print("An error occured while trying to execute output locally {e}.\nAborted...".format(e=e)) + return diff --git a/goex/requirements.txt b/goex/requirements.txt new file mode 100644 index 000000000..def836a35 --- /dev/null +++ b/goex/requirements.txt @@ -0,0 +1,19 @@ +docker +uvicorn +fastapi +docopt==0.6.2 +google_api_python_client==2.118.0 +google_auth_oauthlib==1.2.0 +halo==0.0.31 +mysql-connector-python +openai==1.12.0 +pandas==2.2.0 +protobuf==4.25.3 +pymysql==1.1.0 +python-dotenv==1.0.1 +Requests==2.31.0 +simple-colors +cryptography +waitress==3.0.0 +yarg==0.1.9 +questionary==2.0.1 \ No newline at end of file diff --git a/goex/server.py b/goex/server.py new file mode 100644 index 000000000..cc87b01ac --- /dev/null +++ b/goex/server.py @@ -0,0 +1,200 @@ +# ********************************************************* +# * * +# * This server is used to interface with frontend at * +# * https://github.com/gorilla-llm/gorilla-react-frontend.* +# * * +# ********************************************************* + + +from fastapi import FastAPI, HTTPException, Request +from main import ExecutionEngine, PythonAPIExecutor +import re +import requests +from urllib.parse import quote, urlencode +from google_auth_oauthlib.flow import InstalledAppFlow +from fastapi.middleware.cors import CORSMiddleware +import exec_engine.credentials.credentials_utils as credentials_utils +from exec_engine.utils import SQL_Type, RESTful_Type, Filesystem_Type +import json +import base64 + +app = FastAPI() +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allow_headers=["*"], +) + +@app.options("/{path:path}") +async def options_handler(request: Request, path: str): + # This function intercepts OPTIONS requests and returns the appropriate response + #return {"method": "OPTIONS"} + return {} + +########## GOOGLE AUTH ################################ +SCOPES = [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/calendar.readonly", + "openid", + "https://www.googleapis.com/auth/gmail.compose" , + "https://www.googleapis.com/auth/userinfo.profile", + "https://www.googleapis.com/auth/gmail.readonly", +] +FRONTEND_GMAIL_CREDENTIAL_FILE = "authorizations/credentials/frontend_gmail_credentials.json" +CLI_GMAIL_CREDENTIAL_FILE = "authorizations/credentials/gmail_credentials.json" +frontend_flow = InstalledAppFlow.from_client_secrets_file( + FRONTEND_GMAIL_CREDENTIAL_FILE, SCOPES) +frontend_flow.redirect_uri = 'postmessage' +cli_flow = InstalledAppFlow.from_client_secrets_file( + CLI_GMAIL_CREDENTIAL_FILE, SCOPES) +########## GOOGLE AUTH END ############################## + + + +########## CRED HELPER FUNCTIONS ###################### +def getGoogleCredsFrontend(code, redirect_uri): + frontend_flow.fetch_token(code = code) + return json.loads(frontend_flow.credentials.to_json()) +def getGoogleCredsCli(code, redirect_uri): + cli_flow.redirect_uri = redirect_uri + cli_flow.fetch_token(code = code) + return json.loads(cli_flow.credentials.to_json()) + +SLACK_CREDENTIAL_FILE = "authorizations/credentials/slack_credentials.json" +def getSlackCreds(code, redirect_uri): + with open(SLACK_CREDENTIAL_FILE) as json_file: + client_data = json.load(json_file)["installed"] + data = {"code": code, + "client_id": client_data["client_id"], + "client_secret": client_data["client_secret"], + "redirect_uri":redirect_uri + } + headers = {"content-type": "application/x-www-form-urlencoded;"} + r = requests.post("https://slack.com/api/oauth.v2.access", + data=data, + headers=headers) + return {'token': r.json()['authed_user']['access_token']} + +DISCORD_CREDENTIAL_FILE = "authorizations/credentials/discord_credentials.json" +def getDiscordCreds(code, redirect_uri): + with open(DISCORD_CREDENTIAL_FILE) as json_file: + client_data = json.load(json_file)["installed"] + data = {"grant_type":"authorization_code", "code": code, "redirect_uri":redirect_uri} + url = "https://discord.com/api/oauth2/token" + headers = {"Content-Type": "application/x-www-form-urlencoded"} + r = requests.post(url, + data=data, + headers=headers, + auth=(client_data['client_id'], client_data['client_secret'])) + return {'token' : r.json()['access_token']} + +SPOTIFY_CREDENTIAL_FILE = "authorizations/credentials/spotify_credentials.json" +def getSpotifyCreds(code, redirect_uri): + with open(SPOTIFY_CREDENTIAL_FILE) as json_file: + client_data = json.load(json_file)["installed"] + data = {"code": code, "client_id":client_data['client_id'], "grant_type": 'authorization_code', "redirect_uri": redirect_uri} + form_body = urlencode(data) + url = "https://accounts.spotify.com/api/token" + headers = {"content-type": "application/x-www-form-urlencoded", "Authorization": 'Basic ' + base64.b64encode((f"{client_data['client_id']}:{client_data['client_secret']}").encode()).decode()} + r = requests.post(url, + data=form_body, + headers=headers) + return {'token' : r.json()['access_token']} + +DROPBOX_CREDENTIAL_FILE = "authorizations/credentials/dropbox_credentials.json" +def getDropboxCreds(code, redirect_uri): + with open(DROPBOX_CREDENTIAL_FILE) as json_file: + client_data = json.load(json_file)["installed"] + data = {"code": code, "client_id":client_data['client_id'], 'client_secret' : client_data['client_secret'],"grant_type": 'authorization_code', "redirect_uri": redirect_uri} + form_body = urlencode(data) + url = client_data['token_uri'] + r = requests.post(url, data=form_body) + return {'token' : r.json()['access_token']} + +GITHUB_CREDENTIAL_FILE = "authorizations/credentials/github_credentials.json" +def getGithubCreds(code, redirect_uri): + with open(GITHUB_CREDENTIAL_FILE) as json_file: + client_data = json.load(json_file)["installed"] + params = { + "code": code, + "client_id":client_data['client_id'], + "client_secret" : client_data['client_secret'], + "redirect_uri": redirect_uri + } + url = client_data['token_uri'] + headers = {'Accept': 'application/json'} + r = requests.post(url, params=params, headers=headers) + return {'token' : r.json()['access_token']} +########## CRED HELPER FUNCTIONS END ###################### + + + + +@app.post('/authorize') # takes in temp code from google oauth and returns creds +async def authorize(request: Request): + try: + request_content = await request.json() + code = request_content['code'] + service = request_content['service'] + redirect_uri = request_content['redirect_uri'] + if(service== 'gmail'): + credentials = getGoogleCredsFrontend(code, redirect_uri) + elif(service=='gmail-cli'): + credentials = getGoogleCredsCli(code, redirect_uri) + elif(service == 'slack'): + credentials = getSlackCreds(code, redirect_uri) + elif(service == 'discord'): + credentials = getDiscordCreds(code, redirect_uri) + elif(service == 'spotify'): + credentials = getSpotifyCreds(code, redirect_uri) + elif(service == 'dropbox'): + credentials = getDropboxCreds(code, redirect_uri) + elif(service == 'github'): + credentials = getGithubCreds(code, redirect_uri) + else: + raise HTTPException(status_code=404, detail="Service Not Found") + return credentials + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail="unable to execute with error message {e}".format(e=e)) + + + + + +@app.post('/prompt') +async def prompt_engine(request: Request): + print("IN PROMPT") + try: + request_content = await request.json() + creds = request_content['creds'] # burden on frontend to get the right cred + prompt = request_content['prompt'] + engine = ExecutionEngine() # should we be making a new engine for each request? + engine.api_executor = PythonAPIExecutor(engine.docker_sandbox) + forward_call, backward_call = engine.gen_api_pair(prompt, api_type=RESTful_Type, credentials=creds, model="gpt-4-turbo-preview") + return {"forward_call" : forward_call, "backward_call" : backward_call} + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail="unable to execute with error message {e}".format(e=e)) + +@app.post('/execute') +async def execute(request: Request): + try: + request_content = await request.json() + forward_call = request_content['code'] + #creds = request_content['creds'] + engine = ExecutionEngine() # should we be making a new engine for each request? + engine.api_executor = PythonAPIExecutor(engine.docker_sandbox) + output = engine.api_executor.execute_api_call(forward_call) + print(output) + return output + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail="unable to execute with error message {e}".format(e=e)) + + +# if __name__ == "__main__": +# import uvicorn +# uvicorn.run(app, host="0.0.0.0", port=443, log_level="debug", reload=True) diff --git a/goex/setup.py b/goex/setup.py new file mode 100644 index 000000000..e6f91335b --- /dev/null +++ b/goex/setup.py @@ -0,0 +1,46 @@ +# Copyright 2024 https://github.com/ShishirPatil/gorilla +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the pypi setup file for Gorilla Execution Engine project + +from setuptools import setup, find_packages + +with open('requirements.txt') as f: + requirements = f.read().splitlines() + + +setup( + name="goex", + version="0.0.2", + author="Shishir Patil, Tianjun Zhang, Vivian Fang, Noppapon Chalermchockcharoenkit, Roy Huang, Aaron Hao", + author_email="shishirpatil@berkeley.edu", + url="https://github.com/ShishirPatil/gorilla/", + description="Gorilla Execution Engine CLI", + long_description=open("README.md", "r", encoding="utf-8").read(), + long_description_content_type="text/markdown", + py_modules=["cli"], + packages=find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + ], + python_requires='>=3.6', + install_requires=requirements, + entry_points={ + 'console_scripts': [ + 'goex=cli:main', + ], + }, +)