External repo support for marketplace

.gitignore

@@ -17,3 +17,6 @@ workspace/output
 workspace/input
 celerybeat-schedule
 ../bfg-report*
+superagi/tools/marketplace_tools/
+superagi/tools/external_tools/
+tests/unit_tests/resource_manager/test_path

entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Downloads the tools
+# Downloads the tools from marketplace and external tool repositories
 python superagi/tool_manager.py
 
 # Set executable permissions for install_tool_dependencies.sh

install_tool_dependencies.sh

@@ -4,7 +4,7 @@
 pip install -r /app/requirements.txt
 
 # Loop through the tools directories and install their requirements.txt if they exist
-for tool in /app/superagi/tools/* ; do
+for tool in /app/superagi/tools/* /app/superagi/tools/external_tools/* /app/superagi/tools/marketplace_tools/* ; do
   if [ -d "$tool" ] && [ -f "$tool/requirements.txt" ]; then
     echo "Installing requirements for tool: $(basename "$tool")"
     pip install -r "$tool/requirements.txt"

main.py

@@ -42,7 +42,7 @@
 from superagi.controllers.user import router as user_router
 from superagi.controllers.agent_execution_config import router as agent_execution_config
 from superagi.controllers.analytics import router as analytics_router
-from superagi.helper.tool_helper import register_toolkits
+from superagi.helper.tool_helper import register_toolkits, register_marketplace_toolkits
 from superagi.lib.logger import logger
 from superagi.llms.openai import OpenAi
 from superagi.helper.auth import get_current_user
@@ -115,6 +115,7 @@
 
 app.include_router(google_oauth_router, prefix="/google")
 
+
 # in production you can use Settings management
 # from pydantic to get secret key from .env
 class Settings(BaseModel):
@@ -311,17 +312,26 @@ def build_action_based_agents():
         workflow_step2.next_step_id = workflow_step2.id
         session.commit()
 
-    def check_toolkit_registration():
+    def register_toolkit_for_all_organisation():
         organizations = session.query(Organisation).all()
         for organization in organizations:
             register_toolkits(session, organization)
         logger.info("Successfully registered local toolkits for all Organisations!")
 
+    def register_toolkit_for_master_organisation():
+        marketplace_organisation_id = superagi.config.config.get_config("MARKETPLACE_ORGANISATION_ID")
+        marketplace_organisation = session.query(Organisation).filter(
+            Organisation.id == marketplace_organisation_id).first()
+        if marketplace_organisation is not None:
+            register_marketplace_toolkits(session, marketplace_organisation)
+
     build_single_step_agent()
     build_task_based_agents()
     build_action_based_agents()
     if env != "PROD":
-        check_toolkit_registration()
+        register_toolkit_for_all_organisation()
+    else:
+        register_toolkit_for_master_organisation()
     session.close()
 
 
@@ -393,7 +403,6 @@ def github_auth_handler(code: str = Query(...), Authorize: AuthJWT = Depends()):
             db.session.add(user)
             db.session.commit()
             jwt_token = create_access_token(user_email, Authorize)
-
             redirect_url_success = f"{frontend_url}?access_token={jwt_token}"
             return RedirectResponse(url=redirect_url_success)
         else:

superagi/controllers/toolkit.py

@@ -6,10 +6,12 @@
 from fastapi_sqlalchemy import db
 from superagi.config.config import get_config
 from superagi.helper.auth import get_user_organisation
-from superagi.helper.tool_helper import get_readme_content_from_code_link, download_tool,process_files,add_tool_to_json
+from superagi.helper.tool_helper import get_readme_content_from_code_link, download_tool, process_files, \
+    add_tool_to_json
 from superagi.helper.github_helper import GithubHelper
 from superagi.models.organisation import Organisation
 from superagi.models.tool import Tool
+from superagi.models.tool_config import ToolConfig
 from superagi.models.toolkit import Toolkit
 from superagi.types.common import GitHubLinkRequest
 
@@ -20,7 +22,7 @@
 # marketplace_url = "http://localhost:8001/"
 
 
-#For internal use
+# For internal use
 @router.get("/marketplace/list/{page}")
 def get_marketplace_toolkits(
         page: int = 0,
@@ -51,7 +53,8 @@ def get_marketplace_toolkits(
         toolkit.updated_at = toolkit.updated_at.strftime('%d-%b-%Y').upper()
     return toolkits
 
-#For internal use
+
+# For internal use
 @router.get("/marketplace/details/{toolkit_name}")
 def get_marketplace_toolkit_detail(toolkit_name: str):
     """
@@ -66,10 +69,14 @@ def get_marketplace_toolkit_detail(toolkit_name: str):
     """
 
     organisation_id = int(get_config("MARKETPLACE_ORGANISATION_ID"))
-    toolkit = db.session.query(Toolkit).filter(Toolkit.organisation_id == organisation_id, Toolkit.name == toolkit_name).first()
+    toolkit = db.session.query(Toolkit).filter(Toolkit.organisation_id == organisation_id,
+                                               Toolkit.name == toolkit_name).first()
+    toolkit.tools = db.session.query(Tool).filter(Tool.toolkit_id == toolkit.id).all()
+    toolkit.configs = db.session.query(ToolConfig).filter(ToolConfig.toolkit_id == toolkit.id).all()
     return toolkit
 
-#For internal use
+
+# For internal use
 @router.get("/marketplace/readme/{toolkit_name}")
 def get_marketplace_toolkit_readme(toolkit_name: str):
     """
@@ -93,7 +100,8 @@ def get_marketplace_toolkit_readme(toolkit_name: str):
         raise HTTPException(status_code=404, detail='ToolKit not found')
     return get_readme_content_from_code_link(toolkit.tool_code_link)
 
-#For internal use
+
+# For internal use
 @router.get("/marketplace/tools/{toolkit_name}")
 def get_marketplace_toolkit_tools(toolkit_name: str):
     """
@@ -111,7 +119,8 @@ def get_marketplace_toolkit_tools(toolkit_name: str):
     """
 
     organisation_id = int(get_config("MARKETPLACE_ORGANISATION_ID"))
-    toolkit = db.session.query(Toolkit).filter(Toolkit.name == toolkit_name, Toolkit.organisation_id == organisation_id).first()
+    toolkit = db.session.query(Toolkit).filter(Toolkit.name == toolkit_name,
+                                               Toolkit.organisation_id == organisation_id).first()
     if not toolkit:
         raise HTTPException(status_code=404, detail="ToolKit not found")
     tools = db.session.query(Tool).filter(Tool.toolkit_id == toolkit.id).first()
@@ -132,15 +141,18 @@ def install_toolkit_from_marketplace(toolkit_name: str,
         dict: A message indicating the successful installation of the tool kit.
 
     """
-
     # Check if the tool kit exists
     toolkit = Toolkit.fetch_marketplace_detail(search_str="details",
                                                toolkit_name=toolkit_name)
-    # download_and_install_tool(GitHubLinkRequest(github_link=toolkit['tool_code_link']),
-    #                           organisation=organisation)
-    if not GithubHelper.validate_github_link(toolkit['tool_code_link']):
-        raise HTTPException(status_code=400, detail="Invalid Github link")
-    add_tool_to_json(toolkit['tool_code_link'])
+    db_toolkit = Toolkit.add_or_update(session=db.session, name=toolkit['name'], description=toolkit['description'],
+                          tool_code_link=toolkit['tool_code_link'], organisation_id=organisation.id,
+                          show_toolkit=toolkit['show_toolkit'])
+    for tool in toolkit['tools']:
+        Tool.add_or_update(session=db.session, tool_name=tool['name'], description=tool['description'],
+                           folder_name=tool['folder_name'], class_name=tool['class_name'], file_name=tool['file_name'],
+                           toolkit_id=db_toolkit.id)
+    for config in toolkit['configs']:
+        ToolConfig.add_or_update(session=db.session, toolkit_id=db_toolkit.id, key=config['key'], value=config['value'])
     return {"message": "ToolKit installed successfully"}
 
 
@@ -228,6 +240,7 @@ def get_installed_toolkit_readme(toolkit_name: str, organisation: Organisation =
     readme_content = get_readme_content_from_code_link(toolkit.tool_code_link)
     return readme_content
 
+
 # Following APIs will be used to get marketplace related information
 @router.get("/get")
 def handle_marketplace_operations(
@@ -289,4 +302,4 @@ def get_installed_toolkit_list(organisation: Organisation = Depends(get_user_org
         toolkit_tools = db.session.query(Tool).filter(Tool.toolkit_id == toolkit.id).all()
         toolkit.tools = toolkit_tools
 
-    return toolkits
+    return toolkits

superagi/helper/tool_helper.py

@@ -118,13 +118,17 @@ def load_module_from_file(file_path):
     return module
 
 
-def init_tools(folder_path, session, tool_name_to_toolkit):
+def init_tools(folder_paths, session, tool_name_to_toolkit):
     # Iterate over all subfolders
-    for folder_name in os.listdir(folder_path):
-        folder_dir = os.path.join(folder_path, folder_name)
-        # Iterate over all files in the subfolder
-        if os.path.isdir(folder_dir):
-            # sys.path.append(os.path.abspath('superagi/tools/email'))
+    for folder_path in folder_paths:
+        if not os.path.exists(folder_path):
+            continue
+        for folder_name in os.listdir(folder_path):
+            folder_dir = os.path.join(folder_path, folder_name)
+            # Iterate over all files in the subfolder
+            if not os.path.isdir(folder_dir):
+                continue
+                # sys.path.append(os.path.abspath('superagi/tools/email'))
             sys.path.append(folder_dir)
             for file_name in os.listdir(folder_dir):
                 file_path = os.path.join(folder_dir, file_name)
@@ -147,15 +151,19 @@ def update_base_tool_class_info(classes, file_name, folder_name, session, tool_n
                                               description=tool_description)
 
 
-def init_toolkits(code_link, existing_toolkits, folder_path, organisation, session):
+def init_toolkits(code_link, existing_toolkits, folder_paths, organisation, session):
     tool_name_to_toolkit = {}
     new_toolkits = []
     # Iterate over all subfolders
-    for folder_name in os.listdir(folder_path):
-        folder_dir = os.path.join(folder_path, folder_name)
+    for folder_path in folder_paths:
+        if not os.path.exists(folder_path):
+            continue
+        for folder_name in os.listdir(folder_path):
+            folder_dir = os.path.join(folder_path, folder_name)
 
-        if os.path.isdir(folder_dir):
-            # sys.path.append(os.path.abspath('superagi/tools/email'))
+            if not os.path.isdir(folder_dir):
+                continue
+                # sys.path.append(os.path.abspath('superagi/tools/email'))
             sys.path.append(folder_dir)
             # Iterate over all files in the subfolder
             for file_name in os.listdir(folder_dir):
@@ -214,11 +222,11 @@ def update_base_toolkit_info(classes, code_link, folder_name, new_toolkits, orga
     return tool_name_to_toolkit
 
 
-def process_files(folder_path, session, organisation, code_link=None):
+def process_files(folder_paths, session, organisation, code_link=None):
     existing_toolkits = session.query(Toolkit).filter(Toolkit.organisation_id == organisation.id).all()
 
-    tool_name_to_toolkit = init_toolkits(code_link, existing_toolkits, folder_path, organisation, session)
-    init_tools(folder_path, session, tool_name_to_toolkit)
+    tool_name_to_toolkit = init_toolkits(code_link, existing_toolkits, folder_paths, organisation, session)
+    init_tools(folder_paths, session, tool_name_to_toolkit)
 
 
 def get_readme_content_from_code_link(tool_code_link):
@@ -240,13 +248,18 @@ def get_readme_content_from_code_link(tool_code_link):
 
 
 def register_toolkits(session, organisation):
-    folder_path = get_config("TOOLS_DIR")
-    if folder_path is None:
-        folder_path = "superagi/tools"
+    tool_paths = ["superagi/tools", "superagi/tools/external_tools"]
+    # if get_config("ENV", "DEV") == "PROD":
+    #     tool_paths.append("superagi/tools/marketplace_tools")
     if organisation is not None:
-        process_files(folder_path, session, organisation)
-    logger.info(f"Toolkits Registered Successfully for Organisation ID : {organisation.id}!")
+        process_files(tool_paths, session, organisation)
+        logger.info(f"Toolkits Registered Successfully for Organisation ID : {organisation.id}!")
 
+def register_marketplace_toolkits(session, organisation):
+    tool_paths = ["superagi/tools", "superagi/tools/external_tools","superagi/tools/marketplace_tools"]
+    if organisation is not None:
+        process_files(tool_paths, session, organisation)
+        logger.info(f"Marketplace Toolkits Registered Successfully for Organisation ID : {organisation.id}!")
 
 def extract_repo_name(repo_link):
     # Extract the repository name from the link
@@ -273,10 +286,12 @@ def add_tool_to_json(repo_link):
 
 
 def handle_tools_import():
-    folder_path = get_config("TOOLS_DIR")
-    if folder_path is None:
-        folder_path = "superagi/tools"
-    for folder_name in os.listdir(folder_path):
-        folder_dir = os.path.join(folder_path, folder_name)
-        if os.path.isdir(folder_dir):
-            sys.path.append(folder_dir)
+    print("Handling tools import")
+    tool_paths = ["superagi/tools", "superagi/tools/marketplace_tools", "superagi/tools/external_tools"]
+    for tool_path in tool_paths:
+        if not os.path.exists(tool_path):
+            continue
+        for folder_name in os.listdir(tool_path):
+            folder_dir = os.path.join(tool_path, folder_name)
+            if os.path.isdir(folder_dir):
+                sys.path.append(folder_dir)