From fe55a7890555be23e4244f655d90c109790b8471 Mon Sep 17 00:00:00 2001 From: Axel Theorell Date: Wed, 15 Nov 2023 11:16:23 +0100 Subject: [PATCH 1/6] Made the cli_agent more modular to support different workflows --- gpt_engineer/applications/cli/cli_agent.py | 30 ++++++++++++++++++---- gpt_engineer/core/default/lean_agent.py | 2 +- gpt_engineer/core/default/paths.py | 1 + gpt_engineer/core/default/steps.py | 10 ++++++-- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/gpt_engineer/applications/cli/cli_agent.py b/gpt_engineer/applications/cli/cli_agent.py index a81b82b1bb..9991ef4672 100644 --- a/gpt_engineer/applications/cli/cli_agent.py +++ b/gpt_engineer/applications/cli/cli_agent.py @@ -14,8 +14,19 @@ from gpt_engineer.core.default.paths import memory_path, ENTRYPOINT_FILE from gpt_engineer.core.base_agent import BaseAgent from gpt_engineer.applications.cli.learning import human_review +from typing import TypeVar, Callable +code_gen_fn_type = TypeVar( + "code_gen_fn_type", bound=Callable[[AI, str, BaseRepository], Code] +) +execute_entrypoint_fn_type = TypeVar( + "execute_entrypoint_fn_type", bound=Callable[[BaseExecutionEnv, Code], None] +) +improve_fn_type = TypeVar( + "improve_fn_type", bound=Callable[[AI, str, Code, BaseRepository], Code] +) + class CliAgent(BaseAgent): """ The `Agent` class is responsible for managing the lifecycle of code generation and improvement. @@ -62,32 +73,41 @@ def __init__( memory: BaseRepository, execution_env: BaseExecutionEnv, ai: AI = None, + code_gen_fn: code_gen_fn_type = gen_code, + execute_entrypoint_fn: execute_entrypoint_fn_type = execute_entrypoint, + improve_fn: improve_fn_type = improve ): self.memory = memory self.execution_env = execution_env self.ai = ai or AI() + self.code_gen_fn = code_gen_fn + self.execute_entrypoint_fn = execute_entrypoint_fn + self.improve_fn = improve_fn @classmethod - def with_default_config(cls, path: str, ai: AI = None): + def with_default_config(cls, path: str, ai: AI = None, code_gen_fn: code_gen_fn_type = gen_code, execute_entrypoint_fn: execute_entrypoint_fn_type = execute_entrypoint, improve_fn: improve_fn_type = improve): return cls( memory=OnDiskRepository(memory_path(path)), execution_env=OnDiskExecutionEnv(path), ai=ai, + code_gen_fn=code_gen_fn, + execute_entrypoint_fn=execute_entrypoint_fn, + improve_fn=improve_fn ) def init(self, prompt: str) -> Code: - code = gen_code(self.ai, prompt, self.memory) + code = self.code_gen_fn(self.ai, prompt, self.memory) entrypoint = gen_entrypoint(self.ai, code, self.memory) code = Code(code | entrypoint) - execute_entrypoint(self.execution_env, code) + self.execute_entrypoint_fn(self.execution_env, code) human_review(self.memory) return code def improve(self, prompt: str, code: Code) -> Code: - code = improve(self.ai, prompt, code) + code = self.improve_fn(self.ai, prompt, code, self.memory) if not ENTRYPOINT_FILE in code: entrypoint = gen_entrypoint(self.ai, code, self.memory) code = Code(code | entrypoint) - execute_entrypoint(self.execution_env, code) + self.execute_entrypoint_fn(self.execution_env, code) human_review(self.memory) return code diff --git a/gpt_engineer/core/default/lean_agent.py b/gpt_engineer/core/default/lean_agent.py index 98d6775fb4..c06ce019ec 100644 --- a/gpt_engineer/core/default/lean_agent.py +++ b/gpt_engineer/core/default/lean_agent.py @@ -40,7 +40,7 @@ def init(self, prompt: str) -> Code: return code def improve(self, prompt: str, code: Code) -> Code: - code = improve(self.ai, prompt, code) + code = improve(self.ai, prompt, code, self.memory) if not ENTRYPOINT_FILE in code: entrypoint = gen_entrypoint(self.ai, code, self.memory) code = Code(code | entrypoint) diff --git a/gpt_engineer/core/default/paths.py b/gpt_engineer/core/default/paths.py index d1109053d1..8c9b034f38 100644 --- a/gpt_engineer/core/default/paths.py +++ b/gpt_engineer/core/default/paths.py @@ -3,6 +3,7 @@ META_DATA_REL_PATH = ".gpteng" MEMORY_REL_PATH = os.path.join(META_DATA_REL_PATH, "memory") CODE_GEN_LOG_FILE = "all_output.txt" +IMPROVE_LOG_FILE = "improve.txt" ENTRYPOINT_FILE = "run.sh" ENTRYPOINT_LOG_FILE = "gen_entrypoint_chat.txt" diff --git a/gpt_engineer/core/default/steps.py b/gpt_engineer/core/default/steps.py index 26b8b8ae15..b3ef793ad0 100644 --- a/gpt_engineer/core/default/steps.py +++ b/gpt_engineer/core/default/steps.py @@ -5,6 +5,7 @@ ENTRYPOINT_FILE, CODE_GEN_LOG_FILE, ENTRYPOINT_LOG_FILE, + IMPROVE_LOG_FILE ) from gpt_engineer.core.default.on_disk_repository import OnDiskRepository from gpt_engineer.core.base_repository import BaseRepository @@ -226,7 +227,7 @@ def setup_sys_prompt_existing_code(preprompts: OnDiskRepository) -> str: ) -def improve(ai: AI, prompt: str, code: Code) -> Code: +def improve(ai: AI, prompt: str, code: Code, memory: BaseRepository) -> Code: """ Process and improve the code from a specified set of existing files based on a user prompt. @@ -271,5 +272,10 @@ def improve(ai: AI, prompt: str, code: Code) -> Code: messages = ai.next(messages, step_name=curr_fn()) - overwrite_code_with_edits(messages[-1].content.strip(), code) + chat = messages[-1].content.strip() + + overwrite_code_with_edits(chat, code) + + memory[IMPROVE_LOG_FILE] = chat + return code From 55b1d260cc076a872aeadbc8d7b4ecc45382915e Mon Sep 17 00:00:00 2001 From: Axel Theorell Date: Wed, 15 Nov 2023 11:24:51 +0100 Subject: [PATCH 2/6] added documentation to core/default --- .../core/default/git_version_manager.py | 10 ++ gpt_engineer/core/default/lean_agent.py | 12 ++ .../core/default/on_disk_execution_env.py | 10 ++ .../core/default/on_disk_repository.py | 35 +++++ gpt_engineer/core/default/prompt | 1 + gpt_engineer/core/default/run.sh | 11 ++ gpt_engineer/core/default/steps.py | 138 ++++++------------ prompt | 1 - 8 files changed, 126 insertions(+), 92 deletions(-) create mode 100644 gpt_engineer/core/default/prompt create mode 100644 gpt_engineer/core/default/run.sh delete mode 100644 prompt diff --git a/gpt_engineer/core/default/git_version_manager.py b/gpt_engineer/core/default/git_version_manager.py index 98546ef25a..5ebd83ddeb 100644 --- a/gpt_engineer/core/default/git_version_manager.py +++ b/gpt_engineer/core/default/git_version_manager.py @@ -3,6 +3,16 @@ class GitVersionManager(BaseVersionManager): + """ + Manages versions of code using Git as the version control system. + + This class is responsible for creating snapshots of the code, which typically involves + saving the code to a file and committing it to a Git repository. The snapshot method + returns a unique hash that can be used to identify the version of the code. + + Attributes: + path (str): The file system path where the Git repository is located. + """ def __init__(self, path: str): self.path = path diff --git a/gpt_engineer/core/default/lean_agent.py b/gpt_engineer/core/default/lean_agent.py index c06ce019ec..93decd707a 100644 --- a/gpt_engineer/core/default/lean_agent.py +++ b/gpt_engineer/core/default/lean_agent.py @@ -14,6 +14,18 @@ class LeanAgent(BaseAgent): + """ + An agent that uses AI to generate and improve code based on a given prompt. + + This agent is capable of initializing a codebase from a prompt and improving an existing + codebase based on user input. It uses an AI model to generate and refine code, and it + interacts with a repository and an execution environment to manage and execute the code. + + Attributes: + memory (BaseRepository): The repository where the code and related data are stored. + execution_env (BaseExecutionEnv): The environment in which the code is executed. + ai (AI): The AI model used for generating and improving code. + """ def __init__( self, memory: BaseRepository, diff --git a/gpt_engineer/core/default/on_disk_execution_env.py b/gpt_engineer/core/default/on_disk_execution_env.py index ef5aea92ab..7d0528fe5a 100644 --- a/gpt_engineer/core/default/on_disk_execution_env.py +++ b/gpt_engineer/core/default/on_disk_execution_env.py @@ -6,6 +6,16 @@ class OnDiskExecutionEnv(BaseExecutionEnv): + """ + An execution environment that runs code on the local file system. + + This class is responsible for executing code that is stored on disk. It ensures that + the necessary entrypoint file exists and then runs the code using a subprocess. If the + execution is interrupted by the user, it handles the interruption gracefully. + + Attributes: + path (str): The file system path where the code is located and will be executed. + """ def __init__(self, path: str): self.path = path diff --git a/gpt_engineer/core/default/on_disk_repository.py b/gpt_engineer/core/default/on_disk_repository.py index a0ce5ff98e..c1304c11a2 100644 --- a/gpt_engineer/core/default/on_disk_repository.py +++ b/gpt_engineer/core/default/on_disk_repository.py @@ -45,6 +45,16 @@ class OnDiskRepository(BaseRepository): facilitate CRUD-like interactions. It allows for quick checks on the existence of keys, retrieval of values based on keys, and setting new key-value pairs. + Attributes: + path (Path): The directory path where the database files are stored. + """ + """ + A file-based key-value store where keys correspond to filenames and values to file contents. + + This class provides an interface to a file-based database, leveraging file operations to + facilitate CRUD-like interactions. It allows for quick checks on the existence of keys, + retrieval of values based on keys, and setting new key-value pairs. + Attributes ---------- path : Path @@ -228,6 +238,21 @@ def to_path_list_string(self, supported_code_files_only: bool = False) -> str: # dataclass for all dbs: # @dataclass class FileRepositories: + """ + Encapsulates multiple file-based repositories representing different aspects of a project. + + This class holds references to various repositories used for storing different types of + data, such as memory, logs, preprompts, input, workspace, archive, and project metadata. + + Attributes: + memory (BaseRepository): The repository for storing memory-related data. + logs (BaseRepository): The repository for storing log files. + preprompts (BaseRepository): The repository for storing preprompt data. + input (BaseRepository): The repository for storing input data. + workspace (BaseRepository): The repository representing the workspace. + archive (BaseRepository): The repository for archiving data. + project_metadata (BaseRepository): The repository for storing project metadata. + """ memory: BaseRepository logs: BaseRepository preprompts: BaseRepository @@ -238,6 +263,16 @@ class FileRepositories: def archive(dbs: FileRepositories) -> None: + """ + Archives the contents of memory and workspace repositories. + + This function moves the contents of the memory and workspace repositories to the archive + repository, organizing them with a timestamp. It is used to preserve the state of these + repositories at a specific point in time. + + Parameters: + dbs (FileRepositories): The collection of repositories to be archived. + """ """ Archive the memory and workspace databases. diff --git a/gpt_engineer/core/default/prompt b/gpt_engineer/core/default/prompt new file mode 100644 index 0000000000..8821ce084d --- /dev/null +++ b/gpt_engineer/core/default/prompt @@ -0,0 +1 @@ +go through all functions and classes and check if there is a documentation string. If there is none, add one. If there is one but its not up to date with the code, make an updated version. Make sure to never replace any code. The function of the program is not allowed to change \ No newline at end of file diff --git a/gpt_engineer/core/default/run.sh b/gpt_engineer/core/default/run.sh new file mode 100644 index 0000000000..9e56bc7f91 --- /dev/null +++ b/gpt_engineer/core/default/run.sh @@ -0,0 +1,11 @@ +# Install dependencies +pip install -r requirements.txt + +# Run the necessary parts of the codebase +python git_version_manager.py & +python lean_agent.py & +python on_disk_execution_env.py & +python on_disk_repository.py & +python paths.py & +python steps.py & +wait diff --git a/gpt_engineer/core/default/steps.py b/gpt_engineer/core/default/steps.py index b3ef793ad0..4ace2993bd 100644 --- a/gpt_engineer/core/default/steps.py +++ b/gpt_engineer/core/default/steps.py @@ -26,13 +26,11 @@ def curr_fn() -> str: """ Retrieves the name of the calling function. - This function uses Python's inspection capabilities to dynamically fetch the - name of the function that called `curr_fn()`. This approach ensures that the - function's name isn't hardcoded, making it more resilient to refactoring and - changes to function names. + This utility function uses Python's inspection capabilities to dynamically fetch the + name of the function that called `curr_fn()`. It is used for tracking and logging purposes. Returns: - - str: The name of the function that called `curr_fn()`. + str: The name of the function that called `curr_fn()`. """ return inspect.stack()[1].function @@ -41,16 +39,15 @@ def setup_sys_prompt(preprompts: OnDiskRepository) -> str: """ Constructs a system prompt for the AI based on predefined instructions and philosophies. - This function is responsible for setting up the system prompts for the AI, instructing - it on how to generate code and the coding philosophy to adhere to. The constructed prompt - consists of the "roadmap", "generate" (with dynamic format replacements), and the coding - "philosophy" taken from the given DBs object. + This function prepares a system prompt that guides the AI in generating code according to + specific instructions and coding philosophies. It combines various components such as the + roadmap, generate instructions, and coding philosophy from the provided repository. Parameters: - - preprompts (DBs): The database object containing pre-defined prompts and instructions. + preprompts (OnDiskRepository): The repository containing predefined prompts and instructions. Returns: - - str: The constructed system prompt for the AI. + str: The constructed system prompt for the AI. """ return ( preprompts["roadmap"] @@ -62,24 +59,19 @@ def setup_sys_prompt(preprompts: OnDiskRepository) -> str: def gen_code(ai: AI, prompt: str, memory: BaseRepository) -> Code: """ - Executes the AI model using the default system prompts and saves the full output to memory and program to disk. + Generates code based on a given prompt using an AI model. - This function prepares the system prompt using the provided database configurations - and then invokes the AI model with this system prompt and the main input prompt. - Once the AI generates the output, this function saves it to the specified workspace. - The AI's execution is tracked using the name of the current function for contextual reference. + This function uses the AI model to generate code from a given prompt. It sets up the system + prompt, invokes the AI model, and then saves the generated output to memory. The generated + code is also saved to disk. Parameters: - - ai (AI): An instance of the AI model. - - preprompts (DBs): An instance containing the database configurations, including system and - input prompts, and file formatting preferences. + ai (AI): The AI model used for code generation. + prompt (str): The user prompt that describes what code to generate. + memory (BaseRepository): The repository where the generated code and full output are saved. Returns: - - List[Message]: A list of message objects encapsulating the AI's generated output. - - Note: - The function assumes the `ai.start` method and the `to_files` utility are correctly - set up and functional. Ensure these prerequisites are in place before invoking `simple_gen`. + Code: A dictionary-like object containing the generated code files. """ preprompts = OnDiskRepository(PREPROMPTS_PATH) messages = ai.start(setup_sys_prompt(preprompts), prompt, step_name=curr_fn()) @@ -92,30 +84,19 @@ def gen_code(ai: AI, prompt: str, memory: BaseRepository) -> Code: def gen_entrypoint(ai: AI, code: Code, memory: BaseRepository) -> Code: """ - Generates an entry point script based on a given codebase's information. + Generates an entry point script for a given codebase. - This function prompts the AI model to generate a series of Unix terminal commands - required to a) install dependencies and b) run all necessary components of a codebase - provided in the workspace. The generated commands are then saved to 'run.sh' in the - workspace. + This function prompts the AI model to generate Unix terminal commands necessary for installing + dependencies and running all parts of the codebase. The generated commands are saved to 'run.sh' + and the interaction is logged. Parameters: - - ai (AI): An instance of the AI model. - - prepromptss (DBs): An instance containing the database configurations and workspace - information, particularly the 'all_output.txt' which contains details about the - codebase on disk. + ai (AI): The AI model used for generating the entry point script. + code (Code): The codebase information used to generate the script. + memory (BaseRepository): The repository where the interaction is logged. Returns: - - List[dict]: A list of messages containing the AI's response. - - Notes: - - The AI is instructed not to install packages globally, use 'sudo', provide - explanatory comments, or use placeholders. Instead, it should use example values - where necessary. - - The function uses regular expressions to extract command blocks from the AI's - response to create the 'run.sh' script. - - It assumes the presence of an 'all_output.txt' file in the specified workspace - that contains information about the codebase. + Code: A dictionary-like object containing the entry point script. """ # ToDo: This should enter the preprompts... messages = ai.start( @@ -147,28 +128,21 @@ def gen_entrypoint(ai: AI, code: Code, memory: BaseRepository) -> Code: def execute_entrypoint(execution_env: BaseExecutionEnv, code: Code) -> None: """ - Executes the specified entry point script (`run.sh`) from a workspace. + Executes the entry point script in a given execution environment. - This function prompts the user to confirm whether they wish to execute a script named - 'run.sh' located in the specified workspace. If the user confirms, the script is - executed using a subprocess. The user is informed that they can interrupt the - execution at any time using ctrl+c. + This function handles user confirmation and execution of the 'run.sh' script. It ensures that + the script exists in the code and then proceeds to execute it, allowing the user to interrupt + the process if necessary. Parameters: - - ai (AI): An instance of the AI model, not directly used in this function but - included for consistency with other functions. + execution_env (BaseExecutionEnv): The environment in which the script is executed. + code (Code): The codebase containing the 'run.sh' script. Returns: - - List[dict]: An empty list. This function does not produce a list of messages - but returns an empty list for consistency with the return type of other related - functions. - - Note: - The function assumes the presence of a 'run.sh' script in the specified workspace. - Ensure the script is available and that it has the appropriate permissions - (e.g., executable) before invoking this function. + None """ + if not ENTRYPOINT_FILE in code: raise FileNotFoundError( "The required entrypoint " + ENTRYPOINT_FILE + " does not exist in the code." @@ -207,18 +181,17 @@ def execute_entrypoint(execution_env: BaseExecutionEnv, code: Code) -> None: def setup_sys_prompt_existing_code(preprompts: OnDiskRepository) -> str: """ - Constructs a system prompt for the AI focused on improving an existing codebase. + Constructs a system prompt for the AI to improve existing code. - This function sets up the system prompts for the AI, guiding it on how to - work with and improve an existing code base. The generated prompt consists - of the "improve" instruction (with dynamic format replacements) and the coding - "philosophy" taken from the given DBs object. + This function prepares a system prompt that instructs the AI on how to approach improving + an existing codebase. It combines the "improve" instruction with the coding philosophy from + the provided repository. Parameters: - - preprompts (DBs): The database object containing pre-defined prompts and instructions. + preprompts (OnDiskRepository): The repository containing predefined prompts and instructions. Returns: - - str: The constructed system prompt focused on existing code improvement for the AI. + str: The constructed system prompt for code improvement. """ return ( preprompts["improve"].replace("FILE_FORMAT", preprompts["file_format"]) @@ -229,38 +202,21 @@ def setup_sys_prompt_existing_code(preprompts: OnDiskRepository) -> str: def improve(ai: AI, prompt: str, code: Code, memory: BaseRepository) -> Code: """ - Process and improve the code from a specified set of existing files based on a user prompt. + Improves existing code based on user input using an AI model. - This function first retrieves the code from the designated files and then formats this - code to be processed by the Language Learning Model (LLM). After setting up the system prompt - for existing code improvements, the files' contents are sent to the LLM. Finally, the user's - prompt detailing desired improvements is passed to the LLM, and the subsequent response - from the LLM is used to overwrite the original files. + This function processes and enhances the code from a set of existing files based on a user + prompt. It formats the code for the AI model, sends it for processing, and then applies the + improvements to the original files. Parameters: - - ai (AI): An instance of the AI model that is responsible for processing and generating - responses based on the provided system and user inputs. - - dbs (DBs): An instance containing the database configurations, user prompts, and project metadata. - It is used to fetch the selected files for improvement and the user's improvement prompt. + ai (AI): The AI model used for code improvement. + prompt (str): The user prompt detailing the desired code improvements. + code (Code): The existing codebase to be improved. + memory (BaseRepository): The repository where the improvement process is logged. Returns: - - list[Message]: Returns a list of Message objects that record the interaction between the - system, user, and the AI model. This includes both the input to and the response from the LLM. - - Notes: - - Ensure that the user has correctly set up the desired files for improvement and provided an - appropriate prompt before calling this function. - - The function expects the files to be formatted in a specific way to be properly processed by the LLM. + Code: A dictionary-like object containing the improved code files. """ - - """ - After the file list and prompt have been aquired, this function is called - to sent the formatted prompt to the LLM. - """ - - # files_info = get_code_strings( - # dbs.workspace, dbs.project_metadata - # ) # this has file names relative to the workspace path db = OnDiskRepository(PREPROMPTS_PATH) messages = [ SystemMessage(content=setup_sys_prompt_existing_code(db)), diff --git a/prompt b/prompt deleted file mode 100644 index 242e8c6595..0000000000 --- a/prompt +++ /dev/null @@ -1 +0,0 @@ -add/update documentation strings on all functions and classes. Make sure that they are up-to-date \ No newline at end of file From 8a002d9fb52d910e91f78de1ea9a9529c5209d46 Mon Sep 17 00:00:00 2001 From: Axel Theorell Date: Wed, 15 Nov 2023 13:04:11 +0100 Subject: [PATCH 3/6] Extracted useful steps from legacy/steps to custom_steps --- gpt_engineer/__init__.py | 1 - gpt_engineer/applications/cli/main.py | 3 - gpt_engineer/core/default/prompt | 1 - gpt_engineer/core/default/run.sh | 11 - gpt_engineer/legacy/steps.py | 545 +------------------------- gpt_engineer/tools/custom_steps.py | 239 +++++++++++ 6 files changed, 240 insertions(+), 560 deletions(-) delete mode 100644 gpt_engineer/core/default/prompt delete mode 100644 gpt_engineer/core/default/run.sh create mode 100644 gpt_engineer/tools/custom_steps.py diff --git a/gpt_engineer/__init__.py b/gpt_engineer/__init__.py index 93a16e00c1..a1b8c21ba1 100644 --- a/gpt_engineer/__init__.py +++ b/gpt_engineer/__init__.py @@ -4,6 +4,5 @@ domain, chat_to_files, ) -from gpt_engineer.legacy import steps from gpt_engineer.tools import code_vector_repository from gpt_engineer.core.default import on_disk_repository diff --git a/gpt_engineer/applications/cli/main.py b/gpt_engineer/applications/cli/main.py index 22a14cc312..9ed3ef17b4 100644 --- a/gpt_engineer/applications/cli/main.py +++ b/gpt_engineer/applications/cli/main.py @@ -86,9 +86,6 @@ def main( project_path: str = typer.Argument("projects/example", help="path"), model: str = typer.Argument("gpt-4-1106-preview", help="model id string"), temperature: float = 0.1, - # steps_config: StepsConfig = typer.Option( - # StepsConfig.DEFAULT, "--steps", "-s", help="decide which steps to run" - # ), improve_mode: bool = typer.Option( False, "--improve", diff --git a/gpt_engineer/core/default/prompt b/gpt_engineer/core/default/prompt deleted file mode 100644 index 8821ce084d..0000000000 --- a/gpt_engineer/core/default/prompt +++ /dev/null @@ -1 +0,0 @@ -go through all functions and classes and check if there is a documentation string. If there is none, add one. If there is one but its not up to date with the code, make an updated version. Make sure to never replace any code. The function of the program is not allowed to change \ No newline at end of file diff --git a/gpt_engineer/core/default/run.sh b/gpt_engineer/core/default/run.sh deleted file mode 100644 index 9e56bc7f91..0000000000 --- a/gpt_engineer/core/default/run.sh +++ /dev/null @@ -1,11 +0,0 @@ -# Install dependencies -pip install -r requirements.txt - -# Run the necessary parts of the codebase -python git_version_manager.py & -python lean_agent.py & -python on_disk_execution_env.py & -python on_disk_repository.py & -python paths.py & -python steps.py & -wait diff --git a/gpt_engineer/legacy/steps.py b/gpt_engineer/legacy/steps.py index 2af1d3040b..a96b76bc90 100644 --- a/gpt_engineer/legacy/steps.py +++ b/gpt_engineer/legacy/steps.py @@ -46,8 +46,6 @@ """ import inspect -import re -import subprocess from enum import Enum from platform import platform @@ -55,20 +53,10 @@ from typing import List, Union from langchain.schema import AIMessage, HumanMessage, SystemMessage -from termcolor import colored -from pathlib import Path from gpt_engineer.core.ai import AI -from gpt_engineer.core.chat_to_files import ( - # format_file_to_input, - # get_code_strings, - overwrite_code_with_edits, - # to_files_and_memory, -) from gpt_engineer.core.default.on_disk_repository import FileRepositories -from gpt_engineer.applications.cli.file_selector import FILE_LIST_NAME, ask_for_files -from gpt_engineer.applications.cli.learning import human_review_input -from gpt_engineer.tools.code_vector_repository import CodeVectorRepository +from gpt_engineer.tools.custom_steps import self_heal, vector_improve, gen_clarified_code, clarify, lite_gen MAX_SELF_HEAL_ATTEMPTS = 2 # constants for self healing code ASSUME_WORKING_TIMEOUT = 30 @@ -89,63 +77,6 @@ def get_platform_info(): return a + b -def get_platform_info(): - """Returns the Platform: OS, and the Python version. - This is used for self healing. There are some possible areas of conflict here if - you use a different version of Python in your virtualenv. A better solution would - be to have this info printed from the virtualenv. - """ - v = version_info - a = f"Python Version: {v.major}.{v.minor}.{v.micro}" - b = f"\nOS: {platform()}\n" - return a + b - - -def setup_sys_prompt(dbs: FileRepositories) -> str: - """ - Constructs a system prompt for the AI based on predefined instructions and philosophies. - - This function is responsible for setting up the system prompts for the AI, instructing - it on how to generate code and the coding philosophy to adhere to. The constructed prompt - consists of the "roadmap", "generate" (with dynamic format replacements), and the coding - "philosophy" taken from the given DBs object. - - Parameters: - - dbs (DBs): The database object containing pre-defined prompts and instructions. - - Returns: - - str: The constructed system prompt for the AI. - """ - return ( - dbs.preprompts["roadmap"] - + dbs.preprompts["generate"].replace("FILE_FORMAT", dbs.preprompts["file_format"]) - + "\nUseful to know:\n" - + dbs.preprompts["philosophy"] - ) - - -def setup_sys_prompt_existing_code(dbs: FileRepositories) -> str: - """ - Constructs a system prompt for the AI focused on improving an existing codebase. - - This function sets up the system prompts for the AI, guiding it on how to - work with and improve an existing code base. The generated prompt consists - of the "improve" instruction (with dynamic format replacements) and the coding - "philosophy" taken from the given DBs object. - - Parameters: - - dbs (DBs): The database object containing pre-defined prompts and instructions. - - Returns: - - str: The constructed system prompt focused on existing code improvement for the AI. - """ - return ( - dbs.preprompts["improve"].replace("FILE_FORMAT", dbs.preprompts["file_format"]) - + "\nUseful to know:\n" - + dbs.preprompts["philosophy"] - ) - - def curr_fn() -> str: """ Retrieves the name of the calling function. @@ -161,34 +92,6 @@ def curr_fn() -> str: return inspect.stack()[1].function -def lite_gen(ai: AI, dbs: FileRepositories) -> List[Message]: - """ - Executes the AI model using the main prompt and saves the generated results. - - This function invokes the AI model by feeding it the main prompt. After the - AI processes and generates the output, the function saves this output to the - specified workspace. The AI's output is also tracked using the current function's - name to provide context. - - Parameters: - - ai (AI): An instance of the AI model. - - dbs (DBs): An instance containing the database configurations, including input prompts - and file formatting preferences. - - Returns: - - List[Message]: A list of message objects encapsulating the AI's output. - - Note: - The function assumes the `ai.start` method and the `to_files` utility to be correctly - set up and functional. Ensure these prerequisites before invoking `lite_gen`. - """ - messages = ai.start( - dbs.input["prompt"], dbs.preprompts["file_format"], step_name=curr_fn() - ) - to_files_and_memory(messages[-1].content.strip(), dbs) - return messages - - def simple_gen(ai: AI, dbs: FileRepositories) -> List[Message]: """ Executes the AI model using the default system prompts and saves the output. @@ -215,215 +118,6 @@ def simple_gen(ai: AI, dbs: FileRepositories) -> List[Message]: return messages -def clarify(ai: AI, dbs: FileRepositories) -> List[Message]: - """ - Interactively queries the user for clarifications on the prompt and saves the AI's responses. - - This function presents a series of clarifying questions to the user, based on the AI's - initial assessment of the provided prompt. The user can continue to interact and seek - clarifications until they indicate that they have "nothing to clarify" or manually - opt to move on. If the user doesn't provide any input, the AI is instructed to make its - own assumptions and to state them explicitly before proceeding. - - Parameters: - - ai (AI): An instance of the AI model. - - dbs (DBs): An instance containing the database configurations, which includes system - and input prompts. - - Returns: - - List[Message]: A list of message objects encapsulating the AI's generated output and - interactions. - - """ - messages: List[Message] = [SystemMessage(content=dbs.preprompts["clarify"])] - user_input = dbs.input["prompt"] - while True: - messages = ai.next(messages, user_input, step_name=curr_fn()) - msg = messages[-1].content.strip() - - if "nothing to clarify" in msg.lower(): - break - - if msg.lower().startswith("no"): - print("Nothing to clarify.") - break - - print() - user_input = input('(answer in text, or "c" to move on)\n') - print() - - if not user_input or user_input == "c": - print("(letting gpt-engineer make its own assumptions)") - print() - messages = ai.next( - messages, - "Make your own assumptions and state them explicitly before starting", - step_name=curr_fn(), - ) - print() - return messages - - user_input += """ - \n\n - Is anything else unclear? If yes, ask another question.\n - Otherwise state: "Nothing to clarify" - """ - - print() - return messages - - -def gen_clarified_code(ai: AI, dbs: FileRepositories) -> List[dict]: - """ - Generates code based on clarifications obtained from the user. - - This function processes the messages logged during the user's clarification session - and uses them, along with the system's prompts, to guide the AI in generating code. - The generated code is saved to a specified workspace. - - Parameters: - - ai (AI): An instance of the AI model, responsible for processing and generating the code. - - dbs (DBs): An instance containing the database configurations, which includes system - and input prompts. - - Returns: - - List[dict]: A list of message dictionaries capturing the AI's interactions and generated - outputs during the code generation process. - """ - messages = AI.deserialize_messages(dbs.logs[clarify.__name__]) - - messages = [ - SystemMessage(content=setup_sys_prompt(dbs)), - ] + messages[ - 1: - ] # skip the first clarify message, which was the original clarify priming prompt - messages = ai.next( - messages, - dbs.preprompts["generate"].replace("FILE_FORMAT", dbs.preprompts["file_format"]), - step_name=curr_fn(), - ) - - to_files_and_memory(messages[-1].content.strip(), dbs) - return messages - - -def execute_entrypoint(ai: AI, dbs: FileRepositories) -> List[dict]: - """ - Executes the specified entry point script (`run.sh`) from a workspace. - - This function prompts the user to confirm whether they wish to execute a script named - 'run.sh' located in the specified workspace. If the user confirms, the script is - executed using a subprocess. The user is informed that they can interrupt the - execution at any time using ctrl+c. - - Parameters: - - ai (AI): An instance of the AI model, not directly used in this function but - included for consistency with other functions. - - dbs (DBs): An instance containing the database configurations and workspace - information. - - Returns: - - List[dict]: An empty list. This function does not produce a list of messages - but returns an empty list for consistency with the return type of other related - functions. - - Note: - The function assumes the presence of a 'run.sh' script in the specified workspace. - Ensure the script is available and that it has the appropriate permissions - (e.g., executable) before invoking this function. - """ - command = dbs.workspace["run.sh"] - - print() - print( - colored( - "Do you want to execute this code? (Y/n)", - "red", - ) - ) - print() - print(command) - print() - if input().lower() not in ["", "y", "yes"]: - print("Ok, not executing the code.") - return [] - print("Executing the code...") - print() - print( - colored( - "Note: If it does not work as expected, consider running the code" - + " in another way than above.", - "green", - ) - ) - print() - print("You can press ctrl+c *once* to stop the execution.") - print() - - p = subprocess.Popen("bash run.sh", shell=True, cwd=dbs.workspace.path) - try: - p.wait() - except KeyboardInterrupt: - print() - print("Stopping execution.") - print("Execution stopped.") - p.kill() - print() - - return [] - - -def gen_entrypoint(ai: AI, dbs: FileRepositories) -> List[dict]: - """ - Generates an entry point script based on a given codebase's information. - - This function prompts the AI model to generate a series of Unix terminal commands - required to a) install dependencies and b) run all necessary components of a codebase - provided in the workspace. The generated commands are then saved to 'run.sh' in the - workspace. - - Parameters: - - ai (AI): An instance of the AI model. - - dbs (DBs): An instance containing the database configurations and workspace - information, particularly the 'all_output.txt' which contains details about the - codebase on disk. - - Returns: - - List[dict]: A list of messages containing the AI's response. - - Notes: - - The AI is instructed not to install packages globally, use 'sudo', provide - explanatory comments, or use placeholders. Instead, it should use example values - where necessary. - - The function uses regular expressions to extract command blocks from the AI's - response to create the 'run.sh' script. - - It assumes the presence of an 'all_output.txt' file in the specified workspace - that contains information about the codebase. - """ - messages = ai.start( - system=( - "You will get information about a codebase that is currently on disk in " - "the current folder.\n" - "From this you will answer with code blocks that includes all the necessary " - "unix terminal commands to " - "a) install dependencies " - "b) run all necessary parts of the codebase (in parallel if necessary).\n" - "Do not install globally. Do not use sudo.\n" - "Do not explain the code, just give the commands.\n" - "Do not use placeholders, use example values (like . for a folder argument) " - "if necessary.\n" - ), - user="Information about the codebase:\n\n" + dbs.memory["all_output.txt"], - step_name=curr_fn(), - ) - print() - - regex = r"```\S*\n(.+?)```" - matches = re.finditer(regex, messages[-1].content.strip(), re.DOTALL) - dbs.workspace["run.sh"] = "\n".join(match.group(1) for match in matches) - return messages - - def use_feedback(ai: AI, dbs: FileRepositories): """ Uses the provided feedback to improve the generated code. @@ -466,66 +160,6 @@ def use_feedback(ai: AI, dbs: FileRepositories): exit(1) -def set_improve_filelist(ai: AI, dbs: FileRepositories): - """ - Set the list of files for the AI to work with in the 'existing code mode'. - - This function initiates the process to determine which files from an existing - codebase the AI should work with. By calling `ask_for_files()`, it prompts for - and sets the specific files that should be considered, storing their full paths. - - Parameters: - - ai (AI): An instance of the AI model. Although passed to this function, it is - not used within the function scope and might be for consistency with other - function signatures. - - dbs (DBs): An instance containing the database configurations and project metadata, - which is used to gather information about the existing codebase. Additionally, - the 'input' is used to handle user interactions related to file selection. - - Returns: - - list: Returns an empty list, which can be utilized for consistency in return - types across related functions. - - Note: - - The selected file paths are stored as a side-effect of calling `ask_for_files()`, - and they aren't directly returned by this function. - """ - """Sets the file list for files to work with in existing code mode.""" - ask_for_files(dbs.project_metadata, dbs.workspace) # stores files as full paths. - return [] - - -def vector_improve(ai: AI, dbs: FileRepositories): - code_vector_repository = CodeVectorRepository() - code_vector_repository.load_from_directory(dbs.workspace.path) - releventDocuments = code_vector_repository.relevent_code_chunks(dbs.input["prompt"]) - - code_file_list = f"Here is a list of all the existing code files present in the root directory your code will be added to:" - code_file_list += "\n {fileRepositories.workspace.to_path_list_string()}" - - relevent_file_contents = f"Here are files relevent to the query which you may like to change, reference or add to \n" - - for doc in releventDocuments: - filename_without_path = Path(doc.metadata["filename"]).name - file_content = dbs.workspace[filename_without_path] - relevent_file_contents += format_file_to_input( - filename_without_path, file_content - ) - - messages = [ - SystemMessage(content=setup_sys_prompt_existing_code(dbs)), - ] - - messages.append(HumanMessage(content=f"{code_file_list}")) - messages.append(HumanMessage(content=f"{relevent_file_contents}")) - messages.append(HumanMessage(content=f"Request: {dbs.input['prompt']}")) - - messages = ai.next(messages, step_name=curr_fn()) - - overwrite_code_with_edits(messages[-1].content.strip(), dbs) - return messages - - def assert_files_ready(ai: AI, dbs: FileRepositories): """ Verify the presence of required files for headless 'improve code' execution. @@ -563,183 +197,6 @@ def assert_files_ready(ai: AI, dbs: FileRepositories): return [] -def get_improve_prompt(ai: AI, dbs: FileRepositories): - """ - Asks the user what they would like to fix. - """ - - if not dbs.input.get("prompt"): - dbs.input["prompt"] = input( - "\nWhat do you need to improve with the selected files?\n" - ) - - confirm_str = "\n".join( - [ - "-----------------------------", - "The following files will be used in the improvement process:", - f"{FILE_LIST_NAME}:", - colored(str(dbs.project_metadata[FILE_LIST_NAME]), "green"), - "", - "The inserted prompt is the following:", - colored(f"{dbs.input['prompt']}", "green"), - "-----------------------------", - "", - "You can change these files in your project before proceeding.", - "", - "Press enter to proceed with modifications.", - "", - ] - ) - input(confirm_str) - return [] - - -def improve_existing_code(ai: AI, dbs: FileRepositories): - """ - Process and improve the code from a specified set of existing files based on a user prompt. - - This function first retrieves the code from the designated files and then formats this - code to be processed by the Language Learning Model (LLM). After setting up the system prompt - for existing code improvements, the files' contents are sent to the LLM. Finally, the user's - prompt detailing desired improvements is passed to the LLM, and the subsequent response - from the LLM is used to overwrite the original files. - - Parameters: - - ai (AI): An instance of the AI model that is responsible for processing and generating - responses based on the provided system and user inputs. - - dbs (DBs): An instance containing the database configurations, user prompts, and project metadata. - It is used to fetch the selected files for improvement and the user's improvement prompt. - - Returns: - - list[Message]: Returns a list of Message objects that record the interaction between the - system, user, and the AI model. This includes both the input to and the response from the LLM. - - Notes: - - Ensure that the user has correctly set up the desired files for improvement and provided an - appropriate prompt before calling this function. - - The function expects the files to be formatted in a specific way to be properly processed by the LLM. - """ - - """ - After the file list and prompt have been aquired, this function is called - to sent the formatted prompt to the LLM. - """ - - files_info = get_code_strings( - dbs.workspace, dbs.project_metadata - ) # this has file names relative to the workspace path - - messages = [ - SystemMessage(content=setup_sys_prompt_existing_code(dbs)), - ] - # Add files as input - for file_name, file_str in files_info.items(): - code_input = format_file_to_input(file_name, file_str) - messages.append(HumanMessage(content=f"{code_input}")) - - messages.append(HumanMessage(content=f"Request: {dbs.input['prompt']}")) - - messages = ai.next(messages, step_name=curr_fn()) - - overwrite_code_with_edits(messages[-1].content.strip(), dbs) - return messages - - -def human_review(ai: AI, dbs: FileRepositories): - """ - Collects human feedback on the code and stores it in memory. - - This function prompts the user for a review of the generated or improved code using the `human_review_input` - function. If a valid review is provided, it's serialized to JSON format and stored within the database's - memory under the "review" key. - - Parameters: - - ai (AI): An instance of the AI model. Although not directly used within the function, it is kept as - a parameter for consistency with other functions. - - dbs (DBs): An instance containing the database configurations, user prompts, project metadata, - and memory storage. This function specifically interacts with the memory storage to save the human review. - - Returns: - - list: Returns an empty list, indicating that there's no subsequent interaction with the LLM - or no further messages to be processed. - - Notes: - - It's assumed that the `human_review_input` function handles all the interactions with the user to - gather feedback and returns either the feedback or None if no feedback was provided. - - Ensure that the database's memory has enough space or is set up correctly to store the serialized review tools. - """ - - """Collects and stores human review of the code""" - review = human_review_input() - if review is not None: - dbs.memory["review"] = review.to_json() # type: ignore - return [] - - -def self_heal(ai: AI, dbs: FileRepositories): - """Attempts to execute the code from the entrypoint and if it fails, - sends the error output back to the AI with instructions to fix. - This code will make `MAX_SELF_HEAL_ATTEMPTS` to try and fix the code - before giving up. - This makes the assuption that the previous step was `gen_entrypoint`, - this code could work with `simple_gen`, or `gen_clarified_code` as well. - """ - - # step 1. execute the entrypoint - log_path = dbs.workspace.path / "log.txt" - - attempts = 0 - messages = [] - - while attempts < MAX_SELF_HEAL_ATTEMPTS: - log_file = open(log_path, "w") # wipe clean on every iteration - timed_out = False - - p = subprocess.Popen( # attempt to run the entrypoint - "bash run.sh", - shell=True, - cwd=dbs.workspace.path, - stdout=log_file, - stderr=log_file, - bufsize=0, - ) - try: # timeout if the process actually runs - p.wait(timeout=ASSUME_WORKING_TIMEOUT) - except subprocess.TimeoutExpired: - timed_out = True - print("The process hit a timeout before exiting.") - - # get the result and output - # step 2. if the return code not 0, package and send to the AI - if p.returncode != 0 and not timed_out: - print("run.sh failed. Let's fix it.") - - # pack results in an AI prompt - - # Using the log from the previous step has all the code and - # the gen_entrypoint prompt inside. - if attempts < 1: - messages = AI.deserialize_messages(dbs.logs[gen_entrypoint.__name__]) - messages.append(ai.fuser(get_platform_info())) # add in OS and Py version - - # append the error message - messages.append(ai.fuser(dbs.workspace["log.txt"])) - - messages = ai.next( - messages, dbs.preprompts["file_format_fix"], step_name=curr_fn() - ) - else: # the process did not fail, we are done here. - return messages - - log_file.close() - - # this overwrites the existing files - to_files_and_memory(messages[-1].content.strip(), dbs) - attempts += 1 - - return messages - - class Config(str, Enum): """ Enumeration representing different configuration modes for the code processing system. diff --git a/gpt_engineer/tools/custom_steps.py b/gpt_engineer/tools/custom_steps.py new file mode 100644 index 0000000000..330ff751d7 --- /dev/null +++ b/gpt_engineer/tools/custom_steps.py @@ -0,0 +1,239 @@ +import subprocess +from pathlib import Path +from typing import List + +from langchain.schema import SystemMessage, HumanMessage + +from gpt_engineer.core.ai import AI +from gpt_engineer.core.chat_to_files import overwrite_code_with_edits +from gpt_engineer.core.default.on_disk_repository import OnDiskRepository +from gpt_engineer.core.base_repository import BaseRepository +from gpt_engineer.core.default.paths import ( + ENTRYPOINT_FILE, + CODE_GEN_LOG_FILE, + ENTRYPOINT_LOG_FILE, + IMPROVE_LOG_FILE +) +from gpt_engineer.core.default.steps import curr_fn, PREPROMPTS_PATH +from gpt_engineer.core.chat_to_files import parse_chat +from gpt_engineer.core.code import Code +from gpt_engineer.tools.code_vector_repository import CodeVectorRepository + + +def self_heal(ai: AI, dbs: FileRepositories): + """Attempts to execute the code from the entrypoint and if it fails, + sends the error output back to the AI with instructions to fix. + This code will make `MAX_SELF_HEAL_ATTEMPTS` to try and fix the code + before giving up. + This makes the assuption that the previous step was `gen_entrypoint`, + this code could work with `simple_gen`, or `gen_clarified_code` as well. + """ + + # step 1. execute the entrypoint + log_path = dbs.workspace.path / "log.txt" + + attempts = 0 + messages = [] + + while attempts < MAX_SELF_HEAL_ATTEMPTS: + log_file = open(log_path, "w") # wipe clean on every iteration + timed_out = False + + p = subprocess.Popen( # attempt to run the entrypoint + "bash run.sh", + shell=True, + cwd=dbs.workspace.path, + stdout=log_file, + stderr=log_file, + bufsize=0, + ) + try: # timeout if the process actually runs + p.wait(timeout=ASSUME_WORKING_TIMEOUT) + except subprocess.TimeoutExpired: + timed_out = True + print("The process hit a timeout before exiting.") + + # get the result and output + # step 2. if the return code not 0, package and send to the AI + if p.returncode != 0 and not timed_out: + print("run.sh failed. Let's fix it.") + + # pack results in an AI prompt + + # Using the log from the previous step has all the code and + # the gen_entrypoint prompt inside. + if attempts < 1: + messages = AI.deserialize_messages(dbs.logs[gen_entrypoint.__name__]) + messages.append(ai.fuser(get_platform_info())) # add in OS and Py version + + # append the error message + messages.append(ai.fuser(dbs.workspace["log.txt"])) + + messages = ai.next( + messages, dbs.preprompts["file_format_fix"], step_name=curr_fn() + ) + else: # the process did not fail, we are done here. + return messages + + log_file.close() + + # this overwrites the existing files + to_files_and_memory(messages[-1].content.strip(), dbs) + attempts += 1 + + return messages + + +def vector_improve(ai: AI, dbs: FileRepositories): + code_vector_repository = CodeVectorRepository() + code_vector_repository.load_from_directory(dbs.workspace.path) + releventDocuments = code_vector_repository.relevent_code_chunks(dbs.input["prompt"]) + + code_file_list = f"Here is a list of all the existing code files present in the root directory your code will be added to:" + code_file_list += "\n {fileRepositories.workspace.to_path_list_string()}" + + relevent_file_contents = f"Here are files relevent to the query which you may like to change, reference or add to \n" + + for doc in releventDocuments: + filename_without_path = Path(doc.metadata["filename"]).name + file_content = dbs.workspace[filename_without_path] + relevent_file_contents += format_file_to_input( + filename_without_path, file_content + ) + + messages = [ + SystemMessage(content=setup_sys_prompt_existing_code(dbs)), + ] + + messages.append(HumanMessage(content=f"{code_file_list}")) + messages.append(HumanMessage(content=f"{relevent_file_contents}")) + messages.append(HumanMessage(content=f"Request: {dbs.input['prompt']}")) + + messages = ai.next(messages, step_name=curr_fn()) + + overwrite_code_with_edits(messages[-1].content.strip(), dbs) + return messages + + +def gen_clarified_code(ai: AI, dbs: FileRepositories) -> List[dict]: + """ + Generates code based on clarifications obtained from the user. + + This function processes the messages logged during the user's clarification session + and uses them, along with the system's prompts, to guide the AI in generating code. + The generated code is saved to a specified workspace. + + Parameters: + - ai (AI): An instance of the AI model, responsible for processing and generating the code. + - dbs (DBs): An instance containing the database configurations, which includes system + and input prompts. + + Returns: + - List[dict]: A list of message dictionaries capturing the AI's interactions and generated + outputs during the code generation process. + """ + messages = AI.deserialize_messages(dbs.logs[clarify.__name__]) + + messages = [ + SystemMessage(content=setup_sys_prompt(dbs)), + ] + messages[ + 1: + ] # skip the first clarify message, which was the original clarify priming prompt + messages = ai.next( + messages, + dbs.preprompts["generate"].replace("FILE_FORMAT", dbs.preprompts["file_format"]), + step_name=curr_fn(), + ) + + to_files_and_memory(messages[-1].content.strip(), dbs) + return messages + + +def clarify(ai: AI, dbs: FileRepositories) -> List[Message]: + """ + Interactively queries the user for clarifications on the prompt and saves the AI's responses. + + This function presents a series of clarifying questions to the user, based on the AI's + initial assessment of the provided prompt. The user can continue to interact and seek + clarifications until they indicate that they have "nothing to clarify" or manually + opt to move on. If the user doesn't provide any input, the AI is instructed to make its + own assumptions and to state them explicitly before proceeding. + + Parameters: + - ai (AI): An instance of the AI model. + - dbs (DBs): An instance containing the database configurations, which includes system + and input prompts. + + Returns: + - List[Message]: A list of message objects encapsulating the AI's generated output and + interactions. + + """ + messages: List[Message] = [SystemMessage(content=dbs.preprompts["clarify"])] + user_input = dbs.input["prompt"] + while True: + messages = ai.next(messages, user_input, step_name=curr_fn()) + msg = messages[-1].content.strip() + + if "nothing to clarify" in msg.lower(): + break + + if msg.lower().startswith("no"): + print("Nothing to clarify.") + break + + print() + user_input = input('(answer in text, or "c" to move on)\n') + print() + + if not user_input or user_input == "c": + print("(letting gpt-engineer make its own assumptions)") + print() + messages = ai.next( + messages, + "Make your own assumptions and state them explicitly before starting", + step_name=curr_fn(), + ) + print() + return messages + + user_input += """ + \n\n + Is anything else unclear? If yes, ask another question.\n + Otherwise state: "Nothing to clarify" + """ + + print() + return messages + + +def lite_gen(ai: AI, prompt: str, memory: BaseRepository) -> Code: + """ + Executes the AI model using the main prompt and saves the generated results. + + This function invokes the AI model by feeding it the main prompt. After the + AI processes and generates the output, the function saves this output to the + specified workspace. The AI's output is also tracked using the current function's + name to provide context. + + Parameters: + - ai (AI): An instance of the AI model. + - dbs (DBs): An instance containing the database configurations, including input prompts + and file formatting preferences. + + Returns: + - List[Message]: A list of message objects encapsulating the AI's output. + + Note: + The function assumes the `ai.start` method and the `to_files` utility to be correctly + set up and functional. Ensure these prerequisites before invoking `lite_gen`. + """ + preprompts = OnDiskRepository(PREPROMPTS_PATH) + messages = ai.start( + prompt, preprompts["file_format"], step_name=curr_fn() + ) + chat = messages[-1].content.strip() + memory[CODE_GEN_LOG_FILE] = chat + files = parse_chat(chat) + code = Code({key: val for key, val in files}) + return messages From 54654e1ec40c215420c91a815d0bbb3111005533 Mon Sep 17 00:00:00 2001 From: Axel Theorell Date: Wed, 15 Nov 2023 14:02:03 +0100 Subject: [PATCH 4/6] Added back clarify and lite gen --- gpt_engineer/applications/cli/main.py | 21 +++++--- gpt_engineer/tools/custom_steps.py | 78 +++++++++++---------------- projects/example/prompt | 2 +- tests/tools/test_file_repository.py | 55 +------------------ 4 files changed, 47 insertions(+), 109 deletions(-) diff --git a/gpt_engineer/applications/cli/main.py b/gpt_engineer/applications/cli/main.py index 9ed3ef17b4..751bb7ce84 100644 --- a/gpt_engineer/applications/cli/main.py +++ b/gpt_engineer/applications/cli/main.py @@ -36,11 +36,8 @@ from gpt_engineer.core.ai import AI from gpt_engineer.core.code import Code from gpt_engineer.applications.cli.file_selector import ask_for_files - -# from gpt_engineer.legacy.steps import STEPS, Config as StepsConfig -# from gpt_engineer.applications.cli.collect import collect_learnings -# from gpt_engineer.applications.cli.learning import check_collection_consent -# from gpt_engineer.tools.code_vector_repository import CodeVectorRepository +from gpt_engineer.tools.custom_steps import lite_gen, gen_clarified_code +from gpt_engineer.core.default.steps import gen_code from gpt_engineer.applications.cli.cli_agent import CliAgent @@ -104,6 +101,12 @@ def main( "-l", help="Lite mode - run only the main prompt.", ), + clarify_mode: bool = typer.Option( + False, + "--clarify", + "-c", + help="Lite mode - discuss specification with AI before implementation.", + ), azure_endpoint: str = typer.Option( "", "--azure", @@ -152,7 +155,13 @@ def main( path = Path(project_path) # .absolute() print("Running gpt-engineer in", path, "\n") prompt = load_prompt(OnDiskRepository(path)) - agent = CliAgent.with_default_config(project_path) + if clarify_mode: + code_gen_fn = gen_clarified_code + elif lite_mode: + code_gen_fn = lite_gen + else: + code_gen_fn = gen_code + agent = CliAgent.with_default_config(project_path, code_gen_fn=code_gen_fn) if improve_mode: code = ask_for_files(project_path) agent.improve(prompt, code) diff --git a/gpt_engineer/tools/custom_steps.py b/gpt_engineer/tools/custom_steps.py index 330ff751d7..35dbf8aa2e 100644 --- a/gpt_engineer/tools/custom_steps.py +++ b/gpt_engineer/tools/custom_steps.py @@ -1,8 +1,8 @@ import subprocess from pathlib import Path -from typing import List +from typing import List, Union -from langchain.schema import SystemMessage, HumanMessage +from langchain.schema import SystemMessage, HumanMessage, AIMessage from gpt_engineer.core.ai import AI from gpt_engineer.core.chat_to_files import overwrite_code_with_edits @@ -14,13 +14,14 @@ ENTRYPOINT_LOG_FILE, IMPROVE_LOG_FILE ) -from gpt_engineer.core.default.steps import curr_fn, PREPROMPTS_PATH +from gpt_engineer.core.default.steps import curr_fn, PREPROMPTS_PATH, setup_sys_prompt from gpt_engineer.core.chat_to_files import parse_chat from gpt_engineer.core.code import Code from gpt_engineer.tools.code_vector_repository import CodeVectorRepository +# Type hint for chat messages +Message = Union[AIMessage, HumanMessage, SystemMessage] - -def self_heal(ai: AI, dbs: FileRepositories): +def self_heal(ai: AI, dbs: OnDiskRepository): """Attempts to execute the code from the entrypoint and if it fails, sends the error output back to the AI with instructions to fix. This code will make `MAX_SELF_HEAL_ATTEMPTS` to try and fix the code @@ -84,7 +85,7 @@ def self_heal(ai: AI, dbs: FileRepositories): return messages -def vector_improve(ai: AI, dbs: FileRepositories): +def vector_improve(ai: AI, dbs: OnDiskRepository): code_vector_repository = CodeVectorRepository() code_vector_repository.load_from_directory(dbs.workspace.path) releventDocuments = code_vector_repository.relevent_code_chunks(dbs.input["prompt"]) @@ -115,7 +116,7 @@ def vector_improve(ai: AI, dbs: FileRepositories): return messages -def gen_clarified_code(ai: AI, dbs: FileRepositories) -> List[dict]: +def gen_clarified_code(ai: AI, prompt: str, memory: BaseRepository) -> Code: """ Generates code based on clarifications obtained from the user. @@ -132,45 +133,9 @@ def gen_clarified_code(ai: AI, dbs: FileRepositories) -> List[dict]: - List[dict]: A list of message dictionaries capturing the AI's interactions and generated outputs during the code generation process. """ - messages = AI.deserialize_messages(dbs.logs[clarify.__name__]) - - messages = [ - SystemMessage(content=setup_sys_prompt(dbs)), - ] + messages[ - 1: - ] # skip the first clarify message, which was the original clarify priming prompt - messages = ai.next( - messages, - dbs.preprompts["generate"].replace("FILE_FORMAT", dbs.preprompts["file_format"]), - step_name=curr_fn(), - ) - - to_files_and_memory(messages[-1].content.strip(), dbs) - return messages - - -def clarify(ai: AI, dbs: FileRepositories) -> List[Message]: - """ - Interactively queries the user for clarifications on the prompt and saves the AI's responses. - - This function presents a series of clarifying questions to the user, based on the AI's - initial assessment of the provided prompt. The user can continue to interact and seek - clarifications until they indicate that they have "nothing to clarify" or manually - opt to move on. If the user doesn't provide any input, the AI is instructed to make its - own assumptions and to state them explicitly before proceeding. - - Parameters: - - ai (AI): An instance of the AI model. - - dbs (DBs): An instance containing the database configurations, which includes system - and input prompts. - - Returns: - - List[Message]: A list of message objects encapsulating the AI's generated output and - interactions. - - """ - messages: List[Message] = [SystemMessage(content=dbs.preprompts["clarify"])] - user_input = dbs.input["prompt"] + preprompts = OnDiskRepository(PREPROMPTS_PATH) + messages: List[Message] = [SystemMessage(content=preprompts["clarify"])] + user_input = prompt while True: messages = ai.next(messages, user_input, step_name=curr_fn()) msg = messages[-1].content.strip() @@ -195,7 +160,7 @@ def clarify(ai: AI, dbs: FileRepositories) -> List[Message]: step_name=curr_fn(), ) print() - return messages + user_input += """ \n\n @@ -204,9 +169,26 @@ def clarify(ai: AI, dbs: FileRepositories) -> List[Message]: """ print() + + messages = [ + SystemMessage(content=setup_sys_prompt(preprompts)), + ] + messages[ + 1: + ] # skip the first clarify message, which was the original clarify priming prompt + messages = ai.next( + messages, + preprompts["generate"].replace("FILE_FORMAT", preprompts["file_format"]), + step_name=curr_fn(), + ) + + chat = messages[-1].content.strip() + memory[CODE_GEN_LOG_FILE] = chat + files = parse_chat(chat) + code = Code({key: val for key, val in files}) return messages + def lite_gen(ai: AI, prompt: str, memory: BaseRepository) -> Code: """ Executes the AI model using the main prompt and saves the generated results. @@ -236,4 +218,4 @@ def lite_gen(ai: AI, prompt: str, memory: BaseRepository) -> Code: memory[CODE_GEN_LOG_FILE] = chat files = parse_chat(chat) code = Code({key: val for key, val in files}) - return messages + return code diff --git a/projects/example/prompt b/projects/example/prompt index 70e145ed6d..18d2c72d53 100644 --- a/projects/example/prompt +++ b/projects/example/prompt @@ -1 +1 @@ -write a hello world program +Implement snake in python using an MVC design \ No newline at end of file diff --git a/tests/tools/test_file_repository.py b/tests/tools/test_file_repository.py index 4bbee3630f..6cbbbeaef8 100644 --- a/tests/tools/test_file_repository.py +++ b/tests/tools/test_file_repository.py @@ -2,7 +2,7 @@ from gpt_engineer.core.default.on_disk_repository import ( OnDiskRepository, - FileRepositories, + # FileRepositories, ) @@ -21,33 +21,6 @@ def test_DB_operations(tmp_path): assert val == "test_value" -def test_DBs_initialization(tmp_path): - dir_names = [ - "memory", - "logs", - "preprompts", - "input", - "workspace", - "archive", - "project_metadata", - ] - directories = [tmp_path / name for name in dir_names] - - # Create DB objects - dbs = [OnDiskRepository(dir) for dir in directories] - - # Create DB instance - dbs_instance = FileRepositories(*dbs) - - assert isinstance(dbs_instance.memory, OnDiskRepository) - assert isinstance(dbs_instance.logs, OnDiskRepository) - assert isinstance(dbs_instance.preprompts, OnDiskRepository) - assert isinstance(dbs_instance.input, OnDiskRepository) - assert isinstance(dbs_instance.workspace, OnDiskRepository) - assert isinstance(dbs_instance.archive, OnDiskRepository) - assert isinstance(dbs_instance.project_metadata, OnDiskRepository) - - def test_large_files(tmp_path): db = OnDiskRepository(tmp_path) large_content = "a" * (10**6) # 1MB of tools @@ -101,29 +74,3 @@ def test_error_messages(tmp_path): assert str(e.value) == "val must be str" - -def test_DBs_dataclass_attributes(tmp_path): - dir_names = [ - "memory", - "logs", - "preprompts", - "input", - "workspace", - "archive", - "project_metadata", - ] - directories = [tmp_path / name for name in dir_names] - - # Create DB objects - dbs = [OnDiskRepository(dir) for dir in directories] - - # Create DBs instance - dbs_instance = FileRepositories(*dbs) - - assert dbs_instance.memory == dbs[0] - assert dbs_instance.logs == dbs[1] - assert dbs_instance.preprompts == dbs[2] - assert dbs_instance.input == dbs[3] - assert dbs_instance.workspace == dbs[4] - assert dbs_instance.archive == dbs[5] - assert dbs_instance.project_metadata == dbs[6] From b2224e2519db864abab91189f62044d4e2ec912f Mon Sep 17 00:00:00 2001 From: Axel Theorell Date: Thu, 16 Nov 2023 10:13:32 +0100 Subject: [PATCH 5/6] adapted self-heal, testing to be done --- gpt_engineer/applications/cli/cli_agent.py | 6 +- .../core/default/on_disk_execution_env.py | 4 +- gpt_engineer/core/default/steps.py | 10 ++- gpt_engineer/legacy/steps.py | 11 +-- gpt_engineer/tools/custom_steps.py | 80 ++++++++++++------- 5 files changed, 64 insertions(+), 47 deletions(-) diff --git a/gpt_engineer/applications/cli/cli_agent.py b/gpt_engineer/applications/cli/cli_agent.py index 9991ef4672..0061f97a42 100644 --- a/gpt_engineer/applications/cli/cli_agent.py +++ b/gpt_engineer/applications/cli/cli_agent.py @@ -21,7 +21,7 @@ "code_gen_fn_type", bound=Callable[[AI, str, BaseRepository], Code] ) execute_entrypoint_fn_type = TypeVar( - "execute_entrypoint_fn_type", bound=Callable[[BaseExecutionEnv, Code], None] + "execute_entrypoint_fn_type", bound=Callable[[AI, BaseExecutionEnv, Code], None] ) improve_fn_type = TypeVar( "improve_fn_type", bound=Callable[[AI, str, Code, BaseRepository], Code] @@ -99,7 +99,7 @@ def init(self, prompt: str) -> Code: code = self.code_gen_fn(self.ai, prompt, self.memory) entrypoint = gen_entrypoint(self.ai, code, self.memory) code = Code(code | entrypoint) - self.execute_entrypoint_fn(self.execution_env, code) + self.execute_entrypoint_fn(self.ai, self.execution_env, code) human_review(self.memory) return code @@ -108,6 +108,6 @@ def improve(self, prompt: str, code: Code) -> Code: if not ENTRYPOINT_FILE in code: entrypoint = gen_entrypoint(self.ai, code, self.memory) code = Code(code | entrypoint) - self.execute_entrypoint_fn(self.execution_env, code) + self.execute_entrypoint_fn(self.ai, self.execution_env, code) human_review(self.memory) return code diff --git a/gpt_engineer/core/default/on_disk_execution_env.py b/gpt_engineer/core/default/on_disk_execution_env.py index 7d0528fe5a..981ae06164 100644 --- a/gpt_engineer/core/default/on_disk_execution_env.py +++ b/gpt_engineer/core/default/on_disk_execution_env.py @@ -26,12 +26,12 @@ def execute_program(self, code: Code) -> subprocess.Popen: + ENTRYPOINT_FILE + " does not exist in the code." ) - + # ToDo: The fact that execution is the de-facto way of saving the code to disk presently should change once version manager is implemented. workspace = OnDiskRepository(self.path) for file_name, file_content in code.items(): workspace[file_name] = file_content - p = subprocess.Popen("bash " + ENTRYPOINT_FILE, shell=True, cwd=self.path) + p = subprocess.Popen("bash " + ENTRYPOINT_FILE, shell=True, cwd=self.path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: p.wait() except KeyboardInterrupt: diff --git a/gpt_engineer/core/default/steps.py b/gpt_engineer/core/default/steps.py index 4ace2993bd..dbebb8c468 100644 --- a/gpt_engineer/core/default/steps.py +++ b/gpt_engineer/core/default/steps.py @@ -126,7 +126,7 @@ def gen_entrypoint(ai: AI, code: Code, memory: BaseRepository) -> Code: return entrypoint_code -def execute_entrypoint(execution_env: BaseExecutionEnv, code: Code) -> None: +def execute_entrypoint(ai: AI, execution_env: BaseExecutionEnv, code: Code) -> None: """ Executes the entry point script in a given execution environment. @@ -140,6 +140,10 @@ def execute_entrypoint(execution_env: BaseExecutionEnv, code: Code) -> None: Returns: None + + Parameters + ---------- + ai """ @@ -217,9 +221,9 @@ def improve(ai: AI, prompt: str, code: Code, memory: BaseRepository) -> Code: Returns: Code: A dictionary-like object containing the improved code files. """ - db = OnDiskRepository(PREPROMPTS_PATH) + preprompts = OnDiskRepository(PREPROMPTS_PATH) messages = [ - SystemMessage(content=setup_sys_prompt_existing_code(db)), + SystemMessage(content=setup_sys_prompt_existing_code(preprompts)), ] # Add files as input messages.append(HumanMessage(content=f"{code.to_chat()}")) diff --git a/gpt_engineer/legacy/steps.py b/gpt_engineer/legacy/steps.py index a96b76bc90..b0c5db4b9a 100644 --- a/gpt_engineer/legacy/steps.py +++ b/gpt_engineer/legacy/steps.py @@ -65,16 +65,7 @@ Message = Union[AIMessage, HumanMessage, SystemMessage] -def get_platform_info(): - """Returns the Platform: OS, and the Python version. - This is used for self healing. There are some possible areas of conflict here if - you use a different version of Python in your virtualenv. A better solution would - be to have this info printed from the virtualenv. - """ - v = version_info - a = f"Python Version: {v.major}.{v.minor}.{v.micro}" - b = f"\nOS: {platform()}\n" - return a + b + def curr_fn() -> str: diff --git a/gpt_engineer/tools/custom_steps.py b/gpt_engineer/tools/custom_steps.py index 35dbf8aa2e..39b0e06a18 100644 --- a/gpt_engineer/tools/custom_steps.py +++ b/gpt_engineer/tools/custom_steps.py @@ -1,7 +1,8 @@ import subprocess from pathlib import Path from typing import List, Union - +from platform import platform +from sys import version_info from langchain.schema import SystemMessage, HumanMessage, AIMessage from gpt_engineer.core.ai import AI @@ -17,11 +18,25 @@ from gpt_engineer.core.default.steps import curr_fn, PREPROMPTS_PATH, setup_sys_prompt from gpt_engineer.core.chat_to_files import parse_chat from gpt_engineer.core.code import Code +from gpt_engineer.core.base_execution_env import BaseExecutionEnv from gpt_engineer.tools.code_vector_repository import CodeVectorRepository # Type hint for chat messages Message = Union[AIMessage, HumanMessage, SystemMessage] +MAX_SELF_HEAL_ATTEMPTS = 2 + + +def get_platform_info(): + """Returns the Platform: OS, and the Python version. + This is used for self healing. There are some possible areas of conflict here if + you use a different version of Python in your virtualenv. A better solution would + be to have this info printed from the virtualenv. + """ + v = version_info + a = f"Python Version: {v.major}.{v.minor}.{v.micro}" + b = f"\nOS: {platform()}\n" + return a + b -def self_heal(ai: AI, dbs: OnDiskRepository): +def self_heal(ai: AI, execution_env: BaseExecutionEnv, code: Code) -> Code: """Attempts to execute the code from the entrypoint and if it fails, sends the error output back to the AI with instructions to fix. This code will make `MAX_SELF_HEAL_ATTEMPTS` to try and fix the code @@ -31,32 +46,32 @@ def self_heal(ai: AI, dbs: OnDiskRepository): """ # step 1. execute the entrypoint - log_path = dbs.workspace.path / "log.txt" + # log_path = dbs.workspace.path / "log.txt" attempts = 0 messages = [] - + preprompts = OnDiskRepository(PREPROMPTS_PATH) while attempts < MAX_SELF_HEAL_ATTEMPTS: - log_file = open(log_path, "w") # wipe clean on every iteration - timed_out = False - - p = subprocess.Popen( # attempt to run the entrypoint - "bash run.sh", - shell=True, - cwd=dbs.workspace.path, - stdout=log_file, - stderr=log_file, - bufsize=0, - ) - try: # timeout if the process actually runs - p.wait(timeout=ASSUME_WORKING_TIMEOUT) - except subprocess.TimeoutExpired: - timed_out = True - print("The process hit a timeout before exiting.") - + # log_file = open(log_path, "w") # wipe clean on every iteration + # timed_out = False + + # p = subprocess.Popen( # attempt to run the entrypoint + # "bash run.sh", + # shell=True, + # cwd=dbs.workspace.path, + # stdout=log_file, + # stderr=log_file, + # bufsize=0, + # ) + # try: # timeout if the process actually runs + # p.wait(timeout=ASSUME_WORKING_TIMEOUT) + # except subprocess.TimeoutExpired: + # timed_out = True + # print("The process hit a timeout before exiting.") + process = execution_env.execute_program(code) # get the result and output # step 2. if the return code not 0, package and send to the AI - if p.returncode != 0 and not timed_out: + if process.returncode != 0: print("run.sh failed. Let's fix it.") # pack results in an AI prompt @@ -64,25 +79,32 @@ def self_heal(ai: AI, dbs: OnDiskRepository): # Using the log from the previous step has all the code and # the gen_entrypoint prompt inside. if attempts < 1: - messages = AI.deserialize_messages(dbs.logs[gen_entrypoint.__name__]) + messages = AI.deserialize_messages(code.to_chat()) messages.append(ai.fuser(get_platform_info())) # add in OS and Py version # append the error message - messages.append(ai.fuser(dbs.workspace["log.txt"])) + # Wait for the process to terminate and get stdout and stderr + stdout, stderr = process.communicate() + + # stdout and stderr are bytes, decode them to string if needed + output = stdout.decode('utf-8') + error = stderr.decode('utf-8') + messages.append(ai.fuser(output + "\n " + error)) messages = ai.next( - messages, dbs.preprompts["file_format_fix"], step_name=curr_fn() + messages, preprompts["file_format_fix"], step_name=curr_fn() ) else: # the process did not fail, we are done here. - return messages + return code - log_file.close() + # log_file.close() # this overwrites the existing files - to_files_and_memory(messages[-1].content.strip(), dbs) + # to_files_and_memory(messages[-1].content.strip(), dbs) + code = parse_chat(messages[-1].content.strip()) attempts += 1 - return messages + return code def vector_improve(ai: AI, dbs: OnDiskRepository): From 4bcc8ca5ff743c64b87e823205972110de968ee4 Mon Sep 17 00:00:00 2001 From: Axel Theorell Date: Thu, 16 Nov 2023 14:39:49 +0100 Subject: [PATCH 6/6] Some bugs fixed for self-heal --- gpt_engineer/applications/cli/cli_agent.py | 21 +++--- gpt_engineer/applications/cli/main.py | 21 ++++-- gpt_engineer/core/chat_to_files.py | 1 + gpt_engineer/tools/custom_steps.py | 75 +++++++++++----------- 4 files changed, 66 insertions(+), 52 deletions(-) diff --git a/gpt_engineer/applications/cli/cli_agent.py b/gpt_engineer/applications/cli/cli_agent.py index 0061f97a42..4154b50495 100644 --- a/gpt_engineer/applications/cli/cli_agent.py +++ b/gpt_engineer/applications/cli/cli_agent.py @@ -16,15 +16,14 @@ from gpt_engineer.applications.cli.learning import human_review from typing import TypeVar, Callable - -code_gen_fn_type = TypeVar( - "code_gen_fn_type", bound=Callable[[AI, str, BaseRepository], Code] +CodeGenType = TypeVar( + "CodeGenType", bound=Callable[[AI, str, BaseRepository], Code] ) -execute_entrypoint_fn_type = TypeVar( - "execute_entrypoint_fn_type", bound=Callable[[AI, BaseExecutionEnv, Code], None] +ExecuteEntrypointType = TypeVar( + "ExecuteEntrypointType", bound=Callable[[AI, BaseExecutionEnv, Code], None] ) -improve_fn_type = TypeVar( - "improve_fn_type", bound=Callable[[AI, str, Code, BaseRepository], Code] +ImproveType = TypeVar( + "ImproveType", bound=Callable[[AI, str, Code, BaseRepository], Code] ) class CliAgent(BaseAgent): @@ -73,9 +72,9 @@ def __init__( memory: BaseRepository, execution_env: BaseExecutionEnv, ai: AI = None, - code_gen_fn: code_gen_fn_type = gen_code, - execute_entrypoint_fn: execute_entrypoint_fn_type = execute_entrypoint, - improve_fn: improve_fn_type = improve + code_gen_fn: CodeGenType = gen_code, + execute_entrypoint_fn: ExecuteEntrypointType = execute_entrypoint, + improve_fn: ImproveType = improve ): self.memory = memory self.execution_env = execution_env @@ -85,7 +84,7 @@ def __init__( self.improve_fn = improve_fn @classmethod - def with_default_config(cls, path: str, ai: AI = None, code_gen_fn: code_gen_fn_type = gen_code, execute_entrypoint_fn: execute_entrypoint_fn_type = execute_entrypoint, improve_fn: improve_fn_type = improve): + def with_default_config(cls, path: str, ai: AI = None, code_gen_fn: CodeGenType = gen_code, execute_entrypoint_fn: ExecuteEntrypointType = execute_entrypoint, improve_fn: ImproveType = improve): return cls( memory=OnDiskRepository(memory_path(path)), execution_env=OnDiskExecutionEnv(path), diff --git a/gpt_engineer/applications/cli/main.py b/gpt_engineer/applications/cli/main.py index 751bb7ce84..d2fda5184a 100644 --- a/gpt_engineer/applications/cli/main.py +++ b/gpt_engineer/applications/cli/main.py @@ -36,8 +36,8 @@ from gpt_engineer.core.ai import AI from gpt_engineer.core.code import Code from gpt_engineer.applications.cli.file_selector import ask_for_files -from gpt_engineer.tools.custom_steps import lite_gen, gen_clarified_code -from gpt_engineer.core.default.steps import gen_code +from gpt_engineer.tools.custom_steps import lite_gen, gen_clarified_code, self_heal +from gpt_engineer.core.default.steps import gen_code, execute_entrypoint from gpt_engineer.applications.cli.cli_agent import CliAgent @@ -107,6 +107,12 @@ def main( "-c", help="Lite mode - discuss specification with AI before implementation.", ), + self_heal_mode: bool = typer.Option( + False, + "--self-heal", + "-sh", + help="Lite mode - discuss specification with AI before implementation.", + ), azure_endpoint: str = typer.Option( "", "--azure", @@ -153,15 +159,22 @@ def main( # project_path # ) # resolve the string to a valid path (eg "a/b/../c" to "a/c") path = Path(project_path) # .absolute() - print("Running gpt-engineer in", path, "\n") + print("Running gpt-engineer in", path.absolute(), "\n") prompt = load_prompt(OnDiskRepository(path)) + # configure generation function if clarify_mode: code_gen_fn = gen_clarified_code elif lite_mode: code_gen_fn = lite_gen else: code_gen_fn = gen_code - agent = CliAgent.with_default_config(project_path, code_gen_fn=code_gen_fn) + # configure execution function + if self_heal_mode: + execution_fn = self_heal + else: + execution_fn = execute_entrypoint + + agent = CliAgent.with_default_config(project_path, code_gen_fn=code_gen_fn, execute_entrypoint_fn=execution_fn) if improve_mode: code = ask_for_files(project_path) agent.improve(prompt, code) diff --git a/gpt_engineer/core/chat_to_files.py b/gpt_engineer/core/chat_to_files.py index 5127e50911..fbd0d2d4c9 100644 --- a/gpt_engineer/core/chat_to_files.py +++ b/gpt_engineer/core/chat_to_files.py @@ -85,6 +85,7 @@ def parse_chat(chat) -> List[Tuple[str, str]]: # files.append(("README.md", readme)) # Return the files + # ToDo: Directly return code object return files diff --git a/gpt_engineer/tools/custom_steps.py b/gpt_engineer/tools/custom_steps.py index 39b0e06a18..7973e29ba3 100644 --- a/gpt_engineer/tools/custom_steps.py +++ b/gpt_engineer/tools/custom_steps.py @@ -79,17 +79,17 @@ def self_heal(ai: AI, execution_env: BaseExecutionEnv, code: Code) -> Code: # Using the log from the previous step has all the code and # the gen_entrypoint prompt inside. if attempts < 1: - messages = AI.deserialize_messages(code.to_chat()) - messages.append(ai.fuser(get_platform_info())) # add in OS and Py version - - # append the error message + messages: List[Message] = [SystemMessage(content=code.to_chat())] + # messages.append(ai.fuser(get_platform_info())) # add in OS and Py version WHAT IS ai.fuser()??? + messages.append(SystemMessage(content=get_platform_info())) + # append the error message # Wait for the process to terminate and get stdout and stderr stdout, stderr = process.communicate() # stdout and stderr are bytes, decode them to string if needed output = stdout.decode('utf-8') error = stderr.decode('utf-8') - messages.append(ai.fuser(output + "\n " + error)) + messages.append(SystemMessage(content=output + "\n " + error)) messages = ai.next( messages, preprompts["file_format_fix"], step_name=curr_fn() @@ -101,41 +101,42 @@ def self_heal(ai: AI, execution_env: BaseExecutionEnv, code: Code) -> Code: # this overwrites the existing files # to_files_and_memory(messages[-1].content.strip(), dbs) - code = parse_chat(messages[-1].content.strip()) + files = parse_chat(messages[-1].content.strip()) + code = Code({key: val for key, val in files}) attempts += 1 return code - -def vector_improve(ai: AI, dbs: OnDiskRepository): - code_vector_repository = CodeVectorRepository() - code_vector_repository.load_from_directory(dbs.workspace.path) - releventDocuments = code_vector_repository.relevent_code_chunks(dbs.input["prompt"]) - - code_file_list = f"Here is a list of all the existing code files present in the root directory your code will be added to:" - code_file_list += "\n {fileRepositories.workspace.to_path_list_string()}" - - relevent_file_contents = f"Here are files relevent to the query which you may like to change, reference or add to \n" - - for doc in releventDocuments: - filename_without_path = Path(doc.metadata["filename"]).name - file_content = dbs.workspace[filename_without_path] - relevent_file_contents += format_file_to_input( - filename_without_path, file_content - ) - - messages = [ - SystemMessage(content=setup_sys_prompt_existing_code(dbs)), - ] - - messages.append(HumanMessage(content=f"{code_file_list}")) - messages.append(HumanMessage(content=f"{relevent_file_contents}")) - messages.append(HumanMessage(content=f"Request: {dbs.input['prompt']}")) - - messages = ai.next(messages, step_name=curr_fn()) - - overwrite_code_with_edits(messages[-1].content.strip(), dbs) - return messages +# Todo: Adapt to refactor and code object +# def vector_improve(ai: AI, dbs: OnDiskRepository): +# code_vector_repository = CodeVectorRepository() +# code_vector_repository.load_from_directory(dbs.workspace.path) +# releventDocuments = code_vector_repository.relevent_code_chunks(dbs.input["prompt"]) +# +# code_file_list = f"Here is a list of all the existing code files present in the root directory your code will be added to:" +# code_file_list += "\n {fileRepositories.workspace.to_path_list_string()}" +# +# relevent_file_contents = f"Here are files relevent to the query which you may like to change, reference or add to \n" +# +# for doc in releventDocuments: +# filename_without_path = Path(doc.metadata["filename"]).name +# file_content = dbs.workspace[filename_without_path] +# relevent_file_contents += format_file_to_input( +# filename_without_path, file_content +# ) +# +# messages = [ +# SystemMessage(content=setup_sys_prompt_existing_code(dbs)), +# ] +# +# messages.append(HumanMessage(content=f"{code_file_list}")) +# messages.append(HumanMessage(content=f"{relevent_file_contents}")) +# messages.append(HumanMessage(content=f"Request: {dbs.input['prompt']}")) +# +# messages = ai.next(messages, step_name=curr_fn()) +# +# overwrite_code_with_edits(messages[-1].content.strip(), dbs) +# return messages def gen_clarified_code(ai: AI, prompt: str, memory: BaseRepository) -> Code: @@ -207,7 +208,7 @@ def gen_clarified_code(ai: AI, prompt: str, memory: BaseRepository) -> Code: memory[CODE_GEN_LOG_FILE] = chat files = parse_chat(chat) code = Code({key: val for key, val in files}) - return messages + return code