From 4c09545aeb66b876b83dbc3bf609bba3b13805a1 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 24 Jun 2023 17:45:54 -0300 Subject: [PATCH 01/41] Clarify missing file error --- gpt_engineer/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpt_engineer/db.py b/gpt_engineer/db.py index c3db67c103..0d7e64eed6 100644 --- a/gpt_engineer/db.py +++ b/gpt_engineer/db.py @@ -18,7 +18,7 @@ def __getitem__(self, key): full_path = self.path / key if not full_path.is_file(): - raise KeyError(key) + raise KeyError(f"File '{key}' could not be found in '{self.path}'") with full_path.open("r", encoding="utf-8") as f: return f.read() From 43d67c892e0074fbad2f3e52b9dadce84545d000 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 19:24:05 -0300 Subject: [PATCH 02/41] Add strategy to fix existing code in computer --- gpt_engineer/chat_to_files.py | 22 ++++++++++++++++- gpt_engineer/preprompts/implement_on_existing | 20 ++++++++++++++++ gpt_engineer/steps.py | 24 ++++++++++++++++++- 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 gpt_engineer/preprompts/implement_on_existing diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 8e251cbdfd..15983ab484 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,5 +1,5 @@ import re - +import os def parse_chat(chat): # -> List[Tuple[str, str]]: # Get all ``` blocks and preceding filenames @@ -40,3 +40,23 @@ def to_files(chat, workspace): files = parse_chat(chat) for file_name, file_content in files: workspace[file_name] = file_content + +def getCodeStrings(input): + filesPaths = input["file_list.txt"].split('\n') + filesDict = {} + for filePath in filesPaths: + with open(filePath, 'r') as file: + fileData = file.read() + fileName = os.path.basename(filePath).split('/')[-1] + filesDict[fileName] = fileData + print(filesDict) + return filesDict + +def formatFileToInput(fileName, fileContent): + filestr = f''' + {fileName} + ``` + {fileContent} + ``` + ''' + return filestr \ No newline at end of file diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing new file mode 100644 index 0000000000..613134ed21 --- /dev/null +++ b/gpt_engineer/preprompts/implement_on_existing @@ -0,0 +1,20 @@ +You are a expert developer and you are tasked to work in fixing an issue or creating a new functionaliy in a set of existing codebase. +After reading the implementation request, you will recieve the code one by one to you. +Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconections between each file, function and classes. +Think step by step and reason yourself to the right decisions to make sure we get it right. +The files are organized in the following way: + +FILENAME is the lowercase file name including the file extension, +LANG is the markup code block language for the code's language, and CODE is the code: + +FILENAME +``` +CODE +``` + +After that, implement the requested functionality by using the recieved code as base. +The received code not necessarely will be a standalone implementation, it could be part of a bigger project which you might not have all files. Trust that everything you need to know inside the received files. +Follow the same coding language and framework observed in the received code, using appropriate best practice file naming convention. +Make sure that code in different files are compatible with each other. +Ensure to implement the requested modification, if you are unsure, write a plausible implementation. +Before you finish, double check that all parts of the architecture is present in the files. \ No newline at end of file diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 7a012f4b74..66d06ca39b 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -8,7 +8,7 @@ from termcolor import colored from gpt_engineer.ai import AI -from gpt_engineer.chat_to_files import to_files +from gpt_engineer.chat_to_files import to_files, getCodeStrings, formatFileToInput from gpt_engineer.db import DBs from gpt_engineer.learning import human_input @@ -18,6 +18,11 @@ def setup_sys_prompt(dbs: DBs) -> str: dbs.preprompts["generate"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] ) +def setup_sys_prompt_existing_code(dbs: DBs) -> str: + return ( + dbs.preprompts["implement_on_existing"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] + ) + def get_prompt(dbs: DBs) -> str: """While we migrate we have this fallback getter""" @@ -244,6 +249,21 @@ def use_feedback(ai: AI, dbs: DBs): to_files(messages[-1]["content"], dbs.workspace) return messages +def improve_existing_code(ai: AI, dbs: DBs): + filesInfo = getCodeStrings(dbs.input) + messages = [ + ai.fsystem(setup_sys_prompt_existing_code(dbs)), + ai.fuser(f"Instructions: {dbs.input['prompt']}") + ] + # Add files as input + for filename, filestr in filesInfo.items(): + codeInput = formatFileToInput(filename, filestr) + messages.append(ai.fuser(f"{codeInput}")) + + messages = ai.next(messages) + to_files(messages[-1]["content"], dbs.workspace) + return messages + def fix_code(ai: AI, dbs: DBs): code_output = json.loads(dbs.logs[gen_code.__name__])[-1]["content"] @@ -275,6 +295,7 @@ class Config(str, Enum): EXECUTE_ONLY = "execute_only" EVALUATE = "evaluate" USE_FEEDBACK = "use_feedback" + IMPROVE_CODE = "improve_code" # Different configs of what steps to run @@ -325,6 +346,7 @@ class Config(str, Enum): Config.USE_FEEDBACK: [use_feedback, gen_entrypoint, execute_entrypoint, human_review], Config.EXECUTE_ONLY: [execute_entrypoint], Config.EVALUATE: [execute_entrypoint, human_review], + Config.IMPROVE_CODE: [improve_existing_code] } # Future steps that can be added: From 80791c54d3ff49494db766f20a1cc615f858a2ed Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 19:31:44 -0300 Subject: [PATCH 03/41] add comments and remove debug prints --- gpt_engineer/chat_to_files.py | 3 ++- gpt_engineer/steps.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 15983ab484..bc0acb7ce7 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -41,6 +41,7 @@ def to_files(chat, workspace): for file_name, file_content in files: workspace[file_name] = file_content +# Get code content from a code list def getCodeStrings(input): filesPaths = input["file_list.txt"].split('\n') filesDict = {} @@ -49,9 +50,9 @@ def getCodeStrings(input): fileData = file.read() fileName = os.path.basename(filePath).split('/')[-1] filesDict[fileName] = fileData - print(filesDict) return filesDict +# Format file for inputing to chat prompt def formatFileToInput(fileName, fileContent): filestr = f''' {fileName} diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 66d06ca39b..5b3b10a3ad 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -19,6 +19,9 @@ def setup_sys_prompt(dbs: DBs) -> str: ) def setup_sys_prompt_existing_code(dbs: DBs) -> str: + """ + Similar to code generation, but using an existing code base. + """ return ( dbs.preprompts["implement_on_existing"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] ) @@ -250,6 +253,12 @@ def use_feedback(ai: AI, dbs: DBs): return messages def improve_existing_code(ai: AI, dbs: DBs): + """ + Based on a list of existing files, ask the AI agent to improve, fix or add a new functionality + Necessary to have a 'file_list.txt' and a 'prompt' in the project folder. + The file_list.txt should have the path of the code to be changed + The prompt should have the request for change. + """ filesInfo = getCodeStrings(dbs.input) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), From 11adb98dce05a759618872ef56d76dea46303a0b Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 19:52:56 -0300 Subject: [PATCH 04/41] Format using black --- gpt_engineer/chat_to_files.py | 15 +++++++++------ gpt_engineer/steps.py | 10 +++++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index bc0acb7ce7..52b9473114 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,6 +1,7 @@ import re import os + def parse_chat(chat): # -> List[Tuple[str, str]]: # Get all ``` blocks and preceding filenames regex = r"(\S+)\n\s*```[^\n]*\n(.+?)```" @@ -41,23 +42,25 @@ def to_files(chat, workspace): for file_name, file_content in files: workspace[file_name] = file_content + # Get code content from a code list def getCodeStrings(input): - filesPaths = input["file_list.txt"].split('\n') + filesPaths = input["file_list.txt"].split("\n") filesDict = {} for filePath in filesPaths: - with open(filePath, 'r') as file: + with open(filePath, "r") as file: fileData = file.read() - fileName = os.path.basename(filePath).split('/')[-1] + fileName = os.path.basename(filePath).split("/")[-1] filesDict[fileName] = fileData return filesDict + # Format file for inputing to chat prompt def formatFileToInput(fileName, fileContent): - filestr = f''' + filestr = f""" {fileName} ``` {fileContent} ``` - ''' - return filestr \ No newline at end of file + """ + return filestr diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 5b3b10a3ad..4b3c4cbf90 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -18,12 +18,15 @@ def setup_sys_prompt(dbs: DBs) -> str: dbs.preprompts["generate"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] ) + def setup_sys_prompt_existing_code(dbs: DBs) -> str: """ Similar to code generation, but using an existing code base. """ return ( - dbs.preprompts["implement_on_existing"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] + dbs.preprompts["implement_on_existing"] + + "\nUseful to know:\n" + + dbs.preprompts["philosophy"] ) @@ -252,6 +255,7 @@ def use_feedback(ai: AI, dbs: DBs): to_files(messages[-1]["content"], dbs.workspace) return messages + def improve_existing_code(ai: AI, dbs: DBs): """ Based on a list of existing files, ask the AI agent to improve, fix or add a new functionality @@ -262,7 +266,7 @@ def improve_existing_code(ai: AI, dbs: DBs): filesInfo = getCodeStrings(dbs.input) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), - ai.fuser(f"Instructions: {dbs.input['prompt']}") + ai.fuser(f"Instructions: {dbs.input['prompt']}"), ] # Add files as input for filename, filestr in filesInfo.items(): @@ -355,7 +359,7 @@ class Config(str, Enum): Config.USE_FEEDBACK: [use_feedback, gen_entrypoint, execute_entrypoint, human_review], Config.EXECUTE_ONLY: [execute_entrypoint], Config.EVALUATE: [execute_entrypoint, human_review], - Config.IMPROVE_CODE: [improve_existing_code] + Config.IMPROVE_CODE: [improve_existing_code], } # Future steps that can be added: From a8ea0d032ee20d07a9a951500efac62d2cc026eb Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 20:04:14 -0300 Subject: [PATCH 05/41] lint with ruff --- gpt_engineer/chat_to_files.py | 2 +- gpt_engineer/steps.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 52b9473114..9a46b47298 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,5 +1,5 @@ -import re import os +import re def parse_chat(chat): # -> List[Tuple[str, str]]: diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 4b3c4cbf90..85781979cd 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -8,7 +8,7 @@ from termcolor import colored from gpt_engineer.ai import AI -from gpt_engineer.chat_to_files import to_files, getCodeStrings, formatFileToInput +from gpt_engineer.chat_to_files import formatFileToInput, getCodeStrings, to_files from gpt_engineer.db import DBs from gpt_engineer.learning import human_input @@ -258,7 +258,8 @@ def use_feedback(ai: AI, dbs: DBs): def improve_existing_code(ai: AI, dbs: DBs): """ - Based on a list of existing files, ask the AI agent to improve, fix or add a new functionality + Based on a list of existing files, ask the AI agent to + improve, fix or add a new functionality Necessary to have a 'file_list.txt' and a 'prompt' in the project folder. The file_list.txt should have the path of the code to be changed The prompt should have the request for change. From 493d63b3f62cd5ec6980d5a2b8ea78fe8605758b Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 20:09:10 -0300 Subject: [PATCH 06/41] add new line at end of file --- gpt_engineer/preprompts/implement_on_existing | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing index 613134ed21..cf572517fe 100644 --- a/gpt_engineer/preprompts/implement_on_existing +++ b/gpt_engineer/preprompts/implement_on_existing @@ -1,5 +1,5 @@ You are a expert developer and you are tasked to work in fixing an issue or creating a new functionaliy in a set of existing codebase. -After reading the implementation request, you will recieve the code one by one to you. +After reading the implementation request, you will recieve the code one by one to you. Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconections between each file, function and classes. Think step by step and reason yourself to the right decisions to make sure we get it right. The files are organized in the following way: @@ -17,4 +17,4 @@ The received code not necessarely will be a standalone implementation, it could Follow the same coding language and framework observed in the received code, using appropriate best practice file naming convention. Make sure that code in different files are compatible with each other. Ensure to implement the requested modification, if you are unsure, write a plausible implementation. -Before you finish, double check that all parts of the architecture is present in the files. \ No newline at end of file +Before you finish, double check that all parts of the architecture is present in the files. From 5c3a03475c4447b90102f7822eb98fb78012de40 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sun, 2 Jul 2023 20:26:19 -0300 Subject: [PATCH 07/41] Fix method naming convention and other reviewing details --- gpt_engineer/chat_to_files.py | 4 ++-- gpt_engineer/preprompts/implement_on_existing | 3 +-- gpt_engineer/steps.py | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 9a46b47298..016cbfe36a 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -44,7 +44,7 @@ def to_files(chat, workspace): # Get code content from a code list -def getCodeStrings(input): +def get_code_strings(input): filesPaths = input["file_list.txt"].split("\n") filesDict = {} for filePath in filesPaths: @@ -56,7 +56,7 @@ def getCodeStrings(input): # Format file for inputing to chat prompt -def formatFileToInput(fileName, fileContent): +def format_file_to_input(fileName, fileContent): filestr = f""" {fileName} ``` diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing index cf572517fe..c0a150a2cb 100644 --- a/gpt_engineer/preprompts/implement_on_existing +++ b/gpt_engineer/preprompts/implement_on_existing @@ -4,8 +4,7 @@ Based on the request, you need to comprehend what needs to be modified by unders Think step by step and reason yourself to the right decisions to make sure we get it right. The files are organized in the following way: -FILENAME is the lowercase file name including the file extension, -LANG is the markup code block language for the code's language, and CODE is the code: +FILENAME is the lowercase file name including the file extension and CODE is the code: FILENAME ``` diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 85781979cd..7a363f95ef 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -8,7 +8,7 @@ from termcolor import colored from gpt_engineer.ai import AI -from gpt_engineer.chat_to_files import formatFileToInput, getCodeStrings, to_files +from gpt_engineer.chat_to_files import format_file_to_input, get_code_strings, to_files from gpt_engineer.db import DBs from gpt_engineer.learning import human_input @@ -264,14 +264,14 @@ def improve_existing_code(ai: AI, dbs: DBs): The file_list.txt should have the path of the code to be changed The prompt should have the request for change. """ - filesInfo = getCodeStrings(dbs.input) + filesInfo = get_code_strings(dbs.input) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), ai.fuser(f"Instructions: {dbs.input['prompt']}"), ] # Add files as input for filename, filestr in filesInfo.items(): - codeInput = formatFileToInput(filename, filestr) + codeInput = format_file_to_input(filename, filestr) messages.append(ai.fuser(f"{codeInput}")) messages = ai.next(messages) From 5505ec41dd49eb1e86aa405335f40d7a8fa20b0a Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Mon, 3 Jul 2023 19:16:37 -0300 Subject: [PATCH 08/41] Fix identation and typos --- gpt_engineer/chat_to_files.py | 2 ++ gpt_engineer/preprompts/implement_on_existing | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 016cbfe36a..a55a5a1d90 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -48,8 +48,10 @@ def get_code_strings(input): filesPaths = input["file_list.txt"].split("\n") filesDict = {} for filePath in filesPaths: + fileData = "" with open(filePath, "r") as file: fileData = file.read() + if fileData: fileName = os.path.basename(filePath).split("/")[-1] filesDict[fileName] = fileData return filesDict diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing index c0a150a2cb..5f69ed3e2a 100644 --- a/gpt_engineer/preprompts/implement_on_existing +++ b/gpt_engineer/preprompts/implement_on_existing @@ -1,5 +1,5 @@ You are a expert developer and you are tasked to work in fixing an issue or creating a new functionaliy in a set of existing codebase. -After reading the implementation request, you will recieve the code one by one to you. +After reading the implementation request, you will receive the code one by one to you. Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconections between each file, function and classes. Think step by step and reason yourself to the right decisions to make sure we get it right. The files are organized in the following way: @@ -11,7 +11,7 @@ FILENAME CODE ``` -After that, implement the requested functionality by using the recieved code as base. +After that, implement the requested functionality by using the received code as base. The received code not necessarely will be a standalone implementation, it could be part of a bigger project which you might not have all files. Trust that everything you need to know inside the received files. Follow the same coding language and framework observed in the received code, using appropriate best practice file naming convention. Make sure that code in different files are compatible with each other. From 0308d4e663471616d1bc40cc145a4101f4ff3d36 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sun, 9 Jul 2023 00:43:35 -0300 Subject: [PATCH 09/41] Improve code with .gpteng folder, add file selection window with tkinter --- gpt_engineer/chat_to_files.py | 67 ++++++++++++++++++++++++++++++++--- gpt_engineer/main.py | 25 +++++++++++-- gpt_engineer/steps.py | 39 ++++++++++++++++---- 3 files changed, 117 insertions(+), 14 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index a55a5a1d90..cac1bfd5e4 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,5 +1,7 @@ import os import re +import tkinter as tk +import tkinter.filedialog as fd def parse_chat(chat): # -> List[Tuple[str, str]]: @@ -43,9 +45,36 @@ def to_files(chat, workspace): workspace[file_name] = file_content -# Get code content from a code list -def get_code_strings(input): - filesPaths = input["file_list.txt"].split("\n") +def overwrite_files(chat, dbs, replace_files): + """ + Replace the AI files to the older local files. + """ + dbs.workspace["all_output.txt"] = chat + + files = parse_chat(chat) + for file_name, file_content in files: + # Verify if the file created by the AI agent was in the input list + if file_name in replace_files: + # If the AI created a file from our input list, we replace it. + with open(replace_files[file_name], "w") as text_file: + text_file.write(file_content) + else: + # If the AI create a new file I don't know where to put it yet + # maybe we can think in a smarter solution for this in the future + # like asking the AI where to put it. + # + # by now, just add this to the workspace inside .gpteng folder + print( + f"Could not find file path for '{file_name}', creating file in workspace" + ) + dbs.workspace[file_name] = file_content + + +def get_code_strings(input) -> dict[str, str]: + """ + Read file_list.txt and return file names and its content. + """ + filesPaths = input["file_list.txt"].strip().split("\n") filesDict = {} for filePath in filesPaths: fileData = "" @@ -57,8 +86,10 @@ def get_code_strings(input): return filesDict -# Format file for inputing to chat prompt -def format_file_to_input(fileName, fileContent): +def format_file_to_input(fileName: str, fileContent: str) -> str: + """ + Format a file string to use as input to AI agent + """ filestr = f""" {fileName} ``` @@ -66,3 +97,29 @@ def format_file_to_input(fileName, fileContent): ``` """ return filestr + + +def ask_for_files(input) -> dict[str, str]: + """ + Display a tkinter file selection window to select context files. + Return a dictionary of file_name as key and file_path as value + """ + root = tk.Tk() + root.withdraw() + root.call("wm", "attributes", ".", "-topmost", True) + file_list = list( + fd.askopenfilenames( + parent=root, + initialdir=os.getcwd(), + title="Select relevant files for your change:", + ) + ) + file_list_string = "" + file_path_info = {} + for file_path in file_list: + file_list_string += file_path + "\n" + # Return a dict with key=file_name and value=file_path + file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path + # Write in file_list so the user can edit and remember what was done + input["file_list.txt"] = file_list_string + return file_path_info diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index 9dea702ce4..a096bd6b41 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -1,5 +1,6 @@ import json import logging +import os import shutil from pathlib import Path @@ -24,6 +25,12 @@ def main( steps_config: steps.Config = typer.Option( steps.Config.DEFAULT, "--steps", "-s", help="decide which steps to run" ), + improve_option: bool = typer.Option( + False, + "--improve", + "-i", + help="Improve code from existing project.", + ), verbose: bool = typer.Option(False, "--verbose", "-v"), run_prefix: str = typer.Option( "", @@ -36,6 +43,18 @@ def main( logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO) input_path = Path(project_path).absolute() + + # For the improve option take current project as path and add .gpteng folder + # By now, ignoring the 'project_path' argument + if improve_option: + input_path = Path(os.getcwd()).absolute() / ".gpteng" + input_path.mkdir(parents=True, exist_ok=True) + # The default option for the --improve is the IMPROVE_CODE, not DEFAULT + # I know this looks ugly, not sure if it is the best way to do that... + # we can change that in the future. + if steps_config == steps.Config.DEFAULT: + steps_config = steps.Config.IMPROVE_CODE + memory_path = input_path / f"{run_prefix}memory" workspace_path = input_path / f"{run_prefix}workspace" @@ -59,12 +78,12 @@ def main( preprompts=DB(Path(__file__).parent / "preprompts"), ) - steps = STEPS[steps_config] - for step in steps: + steps_used = STEPS[steps_config] + for step in steps_used: messages = step(ai, dbs) dbs.logs[step.__name__] = json.dumps(messages) - collect_learnings(model, temperature, steps, dbs) + collect_learnings(model, temperature, steps_used, dbs) if __name__ == "__main__": diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 7a363f95ef..c7dfa446cb 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -8,7 +8,13 @@ from termcolor import colored from gpt_engineer.ai import AI -from gpt_engineer.chat_to_files import format_file_to_input, get_code_strings, to_files +from gpt_engineer.chat_to_files import ( + ask_for_files, + format_file_to_input, + get_code_strings, + overwrite_files, + to_files, +) from gpt_engineer.db import DBs from gpt_engineer.learning import human_input @@ -258,13 +264,33 @@ def use_feedback(ai: AI, dbs: DBs): def improve_existing_code(ai: AI, dbs: DBs): """ - Based on a list of existing files, ask the AI agent to + Ask the user for a list of paths, ask the AI agent to improve, fix or add a new functionality - Necessary to have a 'file_list.txt' and a 'prompt' in the project folder. - The file_list.txt should have the path of the code to be changed - The prompt should have the request for change. + A file selection will appear to select the files. + The terminal will ask for the prompt. """ + file_path_info = ask_for_files(dbs.input) filesInfo = get_code_strings(dbs.input) + dbs.input["prompt"] = input( + "\nWhat do you need to improve with the selected files?\n" + ) + + confirmstr = f""" +----------------------------- +The following files will be used in the improvement process: +{dbs.input["file_list.txt"]} + +The inserted prompt is the following: +'{dbs.input['prompt']}' +----------------------------- + +You can change these files in .gpteng folder ({dbs.input.path}) in your project +before proceeding. + +Press enter to proceed with modifications. + +""" + input(confirmstr) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), ai.fuser(f"Instructions: {dbs.input['prompt']}"), @@ -275,7 +301,8 @@ def improve_existing_code(ai: AI, dbs: DBs): messages.append(ai.fuser(f"{codeInput}")) messages = ai.next(messages) - to_files(messages[-1]["content"], dbs.workspace) + # Maybe we should add another step called "replace" or "overwrite" + overwrite_files(messages[-1]["content"], dbs, replace_files=file_path_info) return messages From c733d72ba6650cb125710f5d541ec227f5c57592 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Mon, 10 Jul 2023 23:26:28 -0300 Subject: [PATCH 10/41] add file selector using terminal --- gpt_engineer/chat_to_files.py | 28 ---- gpt_engineer/file_selector.py | 274 ++++++++++++++++++++++++++++++++++ gpt_engineer/steps.py | 2 +- 3 files changed, 275 insertions(+), 29 deletions(-) create mode 100644 gpt_engineer/file_selector.py diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index cac1bfd5e4..4bdd9fc7d1 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,7 +1,5 @@ import os import re -import tkinter as tk -import tkinter.filedialog as fd def parse_chat(chat): # -> List[Tuple[str, str]]: @@ -97,29 +95,3 @@ def format_file_to_input(fileName: str, fileContent: str) -> str: ``` """ return filestr - - -def ask_for_files(input) -> dict[str, str]: - """ - Display a tkinter file selection window to select context files. - Return a dictionary of file_name as key and file_path as value - """ - root = tk.Tk() - root.withdraw() - root.call("wm", "attributes", ".", "-topmost", True) - file_list = list( - fd.askopenfilenames( - parent=root, - initialdir=os.getcwd(), - title="Select relevant files for your change:", - ) - ) - file_list_string = "" - file_path_info = {} - for file_path in file_list: - file_list_string += file_path + "\n" - # Return a dict with key=file_name and value=file_path - file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path - # Write in file_list so the user can edit and remember what was done - input["file_list.txt"] = file_list_string - return file_path_info diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py new file mode 100644 index 0000000000..38a415dee2 --- /dev/null +++ b/gpt_engineer/file_selector.py @@ -0,0 +1,274 @@ +import os +import tkinter as tk +import tkinter.filedialog as fd + +from pathlib import Path +from typing import List, Union + + +class DisplayablePath(object): + """ + A class representing a displayable path in a file explorer. + """ + + display_filename_prefix_middle = "├── " + display_filename_prefix_last = "└── " + display_parent_prefix_middle = " " + display_parent_prefix_last = "│ " + + def __init__( + self, path: Union[str, Path], parent_path: "DisplayablePath", is_last: bool + ): + """ + Initialize a DisplayablePath object. + + Args: + path (Union[str, Path]): The path of the file or directory. + parent_path (DisplayablePath): The parent path of the file or directory. + is_last (bool): Whether the file or directory is the last child of its parent. + """ + self.depth: int = 0 + self.path = Path(str(path)) + self.parent = parent_path + self.is_last = is_last + if self.parent: + self.depth = self.parent.depth + 1 + + @property + def displayname(self) -> str: + """ + Get the display name of the file or directory. + + Returns: + str: The display name. + """ + if self.path.is_dir(): + return self.path.name + "/" + return self.path.name + + @classmethod + def make_tree(cls, root: Union[str, Path], parent=None, is_last=False, criteria=None): + """ + Generate a tree of DisplayablePath objects. + + Args: + root: The root path of the tree. + parent: The parent path of the root path. Defaults to None. + is_last: Whether the root path is the last child of its parent. + criteria: The criteria function to filter the paths. Defaults to None. + + Yields: + DisplayablePath: The DisplayablePath objects in the tree. + """ + root = Path(str(root)) + criteria = criteria or cls._default_criteria + + displayable_root = cls(root, parent, is_last) + yield displayable_root + + children = sorted( + list(path for path in root.iterdir() if criteria(path)), + key=lambda s: str(s).lower(), + ) + count = 1 + for path in children: + is_last = count == len(children) + if path.is_dir(): + yield from cls.make_tree( + path, parent=displayable_root, is_last=is_last, criteria=criteria + ) + else: + yield cls(path, displayable_root, is_last) + count += 1 + + @classmethod + def _default_criteria(cls, path: Path) -> bool: + """ + The default criteria function to filter the paths. + + Args: + path: The path to check. + + Returns: + bool: True if the path should be included, False otherwise. + """ + return True + + def displayable(self) -> str: + """ + Get the displayable string representation of the file or directory. + + Returns: + str: The displayable string representation. + """ + if self.parent is None: + return self.displayname + + _filename_prefix = ( + self.display_filename_prefix_last + if self.is_last + else self.display_filename_prefix_middle + ) + + parts = ["{!s} {!s}".format(_filename_prefix, self.displayname)] + + parent = self.parent + while parent and parent.parent is not None: + parts.append( + self.display_parent_prefix_middle + if parent.is_last + else self.display_parent_prefix_last + ) + parent = parent.parent + + return "".join(reversed(parts)) + + +class TerminalFileSelector: + def __init__(self, root_folder_path: Path) -> None: + self.number_of_selectable_items = 0 + self.selectable_file_paths: dict[int, str] = {} + self.db_paths = DisplayablePath.make_tree( + root_folder_path, parent=None, criteria=self.is_in_ignoring_extentions + ) + + def display(self): + """ + Select files from a directory and display the selected files. + """ + count = 1 + file_path_enumeration = {} + for path in self.db_paths: + n_digits = len(str(count)) + n_spaces = 3 - n_digits + if n_spaces < 0: + # We can only print 1000 aligned files. I think it is decent enough + n_spaces = 0 + spaces_str = " " * n_spaces + if not path.path.is_dir(): + print(f"{count}. {spaces_str}{path.displayable()}") + file_path_enumeration[count] = path.path + count += 1 + else: + # By now we do not accept selecting entire dirs. + # But could add that in the future. Just need to add more functions + # and remove this else block... + number_space = " " * n_digits + print(f"{number_space} {spaces_str}{path.displayable()}") + + self.number_of_selectable_items = count + self.selectable_file_paths = file_path_enumeration + + def ask_for_selection(self) -> List[str]: + user_input = input( + "Select files by entering the numbers separated by spaces or commas: " + ) + user_input = user_input.replace(",", " ") + selected_files = user_input.split() + selected_paths = [] + for file_number_str in selected_files: + try: + file_number = int(file_number_str) + if 1 <= file_number <= self.number_of_selectable_items: + selected_paths.append(str(self.selectable_file_paths[file_number])) + except ValueError: + pass + return selected_paths + + def is_in_ignoring_extentions(self, path: Path) -> bool: + """ + Check if a path is not hidden or in the __pycache__ directory. + + Args: + path: The path to check. + + Returns: + bool: True if the path is not in ignored rules. False otherwise. + """ + is_hidden = not path.name.startswith(".") + is_pycache = "__pycache__" not in path.name + return is_hidden and is_pycache + + +def ask_for_files(db_input) -> dict[str, str]: + use_last_string = "" + selection_number = 1 + if "file_list.txt" in db_input: + can_use_last = True + use_last_string = ( + f"3 - Use previous file list (available at {db_input.path / 'file_list.txt'})" + ) + selectionstr = f""" +How do you want to select the files? +1 - Use terminal +2 - Use a GUI +{use_last_string} + +Select the option and press enter (default=1) : """ + file_path_list = [] + selected_number_str = input(selectionstr) + if selected_number_str: + selection_number = int(selected_number_str) + is_valid_selection = False + if selection_number == 1: + # Open terminal selection + file_path_list = terminal_file_selector() + is_valid_selection = True + elif selection_number == 2: + # Open GUI selection + file_path_list = gui_file_selector() + is_valid_selection = True + else: + if can_use_last: + if selection_number == 3: + # Use previous file list + is_valid_selection = True + if not is_valid_selection: + print("Invalid number. Select a number from the list above.") + return {} + + file_list_string = "" + file_path_info = {} + if not selection_number == 3: + # New files + for file_path in file_path_list: + file_list_string += file_path + "\n" + # Return a dict with key=file_name and value=file_path + file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path + # Write in file_list so the user can edit and remember what was done + db_input["file_list.txt"] = file_list_string + else: + # If using the the previous file list, we dont need to write file_list.txt + file_list_string = db_input["file_list.txt"] + for file_path in file_path_list: + # Return a dict with key=file_name and value=file_path + file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path + + return file_path_info + + +def gui_file_selector() -> List[str]: + """ + Display a tkinter file selection window to select context files. + """ + root = tk.Tk() + root.withdraw() + root.call("wm", "attributes", ".", "-topmost", True) + file_list = list( + fd.askopenfilenames( + parent=root, + initialdir=os.getcwd(), + title="Select files to improve (or give context):", + ) + ) + return file_list + + +def terminal_file_selector() -> List[str]: + """ + Display a terminal file selection to select context files. + """ + file_selector = TerminalFileSelector(Path(os.getcwd())) + file_selector.display() + selected_list = file_selector.ask_for_selection() + return selected_list diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index c7dfa446cb..558c683425 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -9,13 +9,13 @@ from gpt_engineer.ai import AI from gpt_engineer.chat_to_files import ( - ask_for_files, format_file_to_input, get_code_strings, overwrite_files, to_files, ) from gpt_engineer.db import DBs +from gpt_engineer.file_selector import ask_for_files from gpt_engineer.learning import human_input From 387badbc12d27b8c47a659e5f380fc318f5b88d1 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sun, 16 Jul 2023 15:34:53 -0300 Subject: [PATCH 11/41] fix typos and pep8 case rules, change workspace to cwd --- gpt_engineer/chat_to_files.py | 34 +++++++++---------- gpt_engineer/file_selector.py | 27 +++++++++++---- gpt_engineer/main.py | 7 ++-- gpt_engineer/preprompts/implement_on_existing | 6 ++-- gpt_engineer/steps.py | 12 +++---- 5 files changed, 50 insertions(+), 36 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 4bdd9fc7d1..0223a18fd1 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -72,26 +72,26 @@ def get_code_strings(input) -> dict[str, str]: """ Read file_list.txt and return file names and its content. """ - filesPaths = input["file_list.txt"].strip().split("\n") - filesDict = {} - for filePath in filesPaths: - fileData = "" - with open(filePath, "r") as file: - fileData = file.read() - if fileData: - fileName = os.path.basename(filePath).split("/")[-1] - filesDict[fileName] = fileData - return filesDict - - -def format_file_to_input(fileName: str, fileContent: str) -> str: + files_paths = input["file_list.txt"].strip().split("\n") + files_dict = {} + for file_path in files_paths: + file_data = "" + with open(file_path, "r") as file: + file_data = file.read() + if file_data: + file_name = os.path.basename(file_path).split("/")[-1] + files_dict[file_name] = file_data + return files_dict + + +def format_file_to_input(file_name: str, file_content: str) -> str: """ Format a file string to use as input to AI agent """ - filestr = f""" - {fileName} + file_str = f""" + {file_name} ``` - {fileContent} + {file_content} ``` """ - return filestr + return file_str diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py index 38a415dee2..b9033458a5 100644 --- a/gpt_engineer/file_selector.py +++ b/gpt_engineer/file_selector.py @@ -35,7 +35,7 @@ def __init__( self.depth = self.parent.depth + 1 @property - def displayname(self) -> str: + def display_name(self) -> str: """ Get the display name of the file or directory. @@ -102,7 +102,7 @@ def displayable(self) -> str: str: The displayable string representation. """ if self.parent is None: - return self.displayname + return self.display_name _filename_prefix = ( self.display_filename_prefix_last @@ -110,7 +110,7 @@ def displayable(self) -> str: else self.display_filename_prefix_middle ) - parts = ["{!s} {!s}".format(_filename_prefix, self.displayname)] + parts = ["{!s} {!s}".format(_filename_prefix, self.display_name)] parent = self.parent while parent and parent.parent is not None: @@ -129,7 +129,7 @@ def __init__(self, root_folder_path: Path) -> None: self.number_of_selectable_items = 0 self.selectable_file_paths: dict[int, str] = {} self.db_paths = DisplayablePath.make_tree( - root_folder_path, parent=None, criteria=self.is_in_ignoring_extentions + root_folder_path, parent=None, criteria=self.is_in_ignoring_extensions ) def display(self): @@ -160,6 +160,12 @@ def display(self): self.selectable_file_paths = file_path_enumeration def ask_for_selection(self) -> List[str]: + """ + Ask user to select files from the terminal after displaying it + + Returns: + List[str]: list of selected paths + """ user_input = input( "Select files by entering the numbers separated by spaces or commas: " ) @@ -175,7 +181,7 @@ def ask_for_selection(self) -> List[str]: pass return selected_paths - def is_in_ignoring_extentions(self, path: Path) -> bool: + def is_in_ignoring_extensions(self, path: Path) -> bool: """ Check if a path is not hidden or in the __pycache__ directory. @@ -191,6 +197,13 @@ def is_in_ignoring_extentions(self, path: Path) -> bool: def ask_for_files(db_input) -> dict[str, str]: + """ + Ask user to select files to improve. + It can be done by terminal, gui, or using the old selection. + + Returns: + dict[str, str]: Dictionary where key = file name and value = file path + """ use_last_string = "" selection_number = 1 if "file_list.txt" in db_input: @@ -198,7 +211,7 @@ def ask_for_files(db_input) -> dict[str, str]: use_last_string = ( f"3 - Use previous file list (available at {db_input.path / 'file_list.txt'})" ) - selectionstr = f""" + selection_str = f""" How do you want to select the files? 1 - Use terminal 2 - Use a GUI @@ -206,7 +219,7 @@ def ask_for_files(db_input) -> dict[str, str]: Select the option and press enter (default=1) : """ file_path_list = [] - selected_number_str = input(selectionstr) + selected_number_str = input(selection_str) if selected_number_str: selection_number = int(selected_number_str) is_valid_selection = False diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index a096bd6b41..089375eac3 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -44,6 +44,8 @@ def main( input_path = Path(project_path).absolute() + memory_path = input_path / f"{run_prefix}memory" + workspace_path = input_path / f"{run_prefix}workspace" # For the improve option take current project as path and add .gpteng folder # By now, ignoring the 'project_path' argument if improve_option: @@ -54,9 +56,8 @@ def main( # we can change that in the future. if steps_config == steps.Config.DEFAULT: steps_config = steps.Config.IMPROVE_CODE - - memory_path = input_path / f"{run_prefix}memory" - workspace_path = input_path / f"{run_prefix}workspace" + memory_path = input_path / f"{run_prefix}memory" + workspace_path = Path(os.getcwd()).absolute() if delete_existing: # Delete files and subdirectories in paths diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing index 5f69ed3e2a..7f1fafc96e 100644 --- a/gpt_engineer/preprompts/implement_on_existing +++ b/gpt_engineer/preprompts/implement_on_existing @@ -1,6 +1,6 @@ -You are a expert developer and you are tasked to work in fixing an issue or creating a new functionaliy in a set of existing codebase. +You are a expert developer and you are tasked to work in fixing an issue or creating a new functionality in a set of existing codebase. After reading the implementation request, you will receive the code one by one to you. -Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconections between each file, function and classes. +Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconnections between each file, function and classes. Think step by step and reason yourself to the right decisions to make sure we get it right. The files are organized in the following way: @@ -12,7 +12,7 @@ CODE ``` After that, implement the requested functionality by using the received code as base. -The received code not necessarely will be a standalone implementation, it could be part of a bigger project which you might not have all files. Trust that everything you need to know inside the received files. +The received code not necessarily will be a standalone implementation, it could be part of a bigger project which you might not have all files. Trust that everything you need to know inside the received files. Follow the same coding language and framework observed in the received code, using appropriate best practice file naming convention. Make sure that code in different files are compatible with each other. Ensure to implement the requested modification, if you are unsure, write a plausible implementation. diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 558c683425..1e4a98959d 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -270,12 +270,12 @@ def improve_existing_code(ai: AI, dbs: DBs): The terminal will ask for the prompt. """ file_path_info = ask_for_files(dbs.input) - filesInfo = get_code_strings(dbs.input) + files_info = get_code_strings(dbs.input) dbs.input["prompt"] = input( "\nWhat do you need to improve with the selected files?\n" ) - confirmstr = f""" + confirm_str = f""" ----------------------------- The following files will be used in the improvement process: {dbs.input["file_list.txt"]} @@ -290,15 +290,15 @@ def improve_existing_code(ai: AI, dbs: DBs): Press enter to proceed with modifications. """ - input(confirmstr) + input(confirm_str) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), ai.fuser(f"Instructions: {dbs.input['prompt']}"), ] # Add files as input - for filename, filestr in filesInfo.items(): - codeInput = format_file_to_input(filename, filestr) - messages.append(ai.fuser(f"{codeInput}")) + for file_name, file_str in files_info.items(): + code_input = format_file_to_input(file_name, file_str) + messages.append(ai.fuser(f"{code_input}")) messages = ai.next(messages) # Maybe we should add another step called "replace" or "overwrite" From 14e1ff7b3b8a83bdbd97f9c0c653d477ad778458 Mon Sep 17 00:00:00 2001 From: Lucas Klein Date: Tue, 11 Jul 2023 09:14:46 +0200 Subject: [PATCH 12/41] Added better functionality to file selector input, fixed some bugs, misspellings and cleaned up. Applied leomariga suggestions. --- gpt_engineer/.gpteng/file_list.txt | 20 +++++ gpt_engineer/file_selector.py | 118 ++++++++++++++++++----------- requirements.txt | 5 ++ 3 files changed, 98 insertions(+), 45 deletions(-) create mode 100644 gpt_engineer/.gpteng/file_list.txt create mode 100644 requirements.txt diff --git a/gpt_engineer/.gpteng/file_list.txt b/gpt_engineer/.gpteng/file_list.txt new file mode 100644 index 0000000000..78319baab1 --- /dev/null +++ b/gpt_engineer/.gpteng/file_list.txt @@ -0,0 +1,20 @@ +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\__init__.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\ai.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\chat_to_files.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\collect.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\db.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\domain.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\file_selector.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\learning.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\main.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\fix_code +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\generate +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\implement_on_existing +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\philosophy +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\qa +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\respec +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\spec +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\unit_tests +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\use_feedback +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\use_qa +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\steps.py diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py index 38a415dee2..4da8c1b65b 100644 --- a/gpt_engineer/file_selector.py +++ b/gpt_engineer/file_selector.py @@ -1,4 +1,6 @@ import os +import re +import sys import tkinter as tk import tkinter.filedialog as fd @@ -35,7 +37,7 @@ def __init__( self.depth = self.parent.depth + 1 @property - def displayname(self) -> str: + def display_name(self) -> str: """ Get the display name of the file or directory. @@ -102,7 +104,7 @@ def displayable(self) -> str: str: The displayable string representation. """ if self.parent is None: - return self.displayname + return self.display_name _filename_prefix = ( self.display_filename_prefix_last @@ -110,7 +112,7 @@ def displayable(self) -> str: else self.display_filename_prefix_middle ) - parts = ["{!s} {!s}".format(_filename_prefix, self.displayname)] + parts = ["{!s} {!s}".format(_filename_prefix, self.display_name)] parent = self.parent while parent and parent.parent is not None: @@ -124,20 +126,23 @@ def displayable(self) -> str: return "".join(reversed(parts)) + class TerminalFileSelector: def __init__(self, root_folder_path: Path) -> None: self.number_of_selectable_items = 0 self.selectable_file_paths: dict[int, str] = {} + self.file_path_list: list = [] self.db_paths = DisplayablePath.make_tree( - root_folder_path, parent=None, criteria=self.is_in_ignoring_extentions + root_folder_path, parent=None, criteria=self.is_in_ignoring_extensions ) def display(self): """ Select files from a directory and display the selected files. """ - count = 1 + count = 0 file_path_enumeration = {} + file_path_list = [] for path in self.db_paths: n_digits = len(str(count)) n_spaces = 3 - n_digits @@ -148,6 +153,7 @@ def display(self): if not path.path.is_dir(): print(f"{count}. {spaces_str}{path.displayable()}") file_path_enumeration[count] = path.path + file_path_list.append(path.path) count += 1 else: # By now we do not accept selecting entire dirs. @@ -157,88 +163,110 @@ def display(self): print(f"{number_space} {spaces_str}{path.displayable()}") self.number_of_selectable_items = count + self.file_path_list = file_path_list self.selectable_file_paths = file_path_enumeration def ask_for_selection(self) -> List[str]: user_input = input( - "Select files by entering the numbers separated by spaces or commas: " + "\nSelect files by entering the numbers separated by commas/spaces or specify range with a dash.\nExample: 1,2,3-5,7,9,13-15,18,20 (enter 'all' to select everything)\n\nSelect files: " ) - user_input = user_input.replace(",", " ") - selected_files = user_input.split() selected_paths = [] - for file_number_str in selected_files: + regex = r"\d+(-\d+)?([, ]\d+(-\d+)?)*" + + + if user_input.lower() == "all": + selected_paths = self.file_path_list + elif re.match(regex, user_input): try: - file_number = int(file_number_str) - if 1 <= file_number <= self.number_of_selectable_items: - selected_paths.append(str(self.selectable_file_paths[file_number])) + user_input = user_input.replace("", ",") if " " in user_input else user_input + selected_files = user_input.split(",") + for file_number_str in selected_files: + if "-" in file_number_str: + start, end = file_number_str.split("-") + start = int(start) + end = int(end) + for num in range(start, end + 1): + selected_paths.append(str(self.selectable_file_paths[num])) + else: + num = int(file_number_str) + selected_paths.append(str(self.selectable_file_paths[num])) + except ValueError: pass + else: + print("Please use a valid number/series of numbers.\n") + sys.exit(1) + return selected_paths - def is_in_ignoring_extentions(self, path: Path) -> bool: - """ - Check if a path is not hidden or in the __pycache__ directory. - Args: - path: The path to check. +def is_in_ignoring_extensions(path: Path) -> bool: + """ + Check if a path is not hidden or in the __pycache__ directory. - Returns: - bool: True if the path is not in ignored rules. False otherwise. - """ - is_hidden = not path.name.startswith(".") - is_pycache = "__pycache__" not in path.name - return is_hidden and is_pycache + Args: + path: The path to check. + + Returns: + bool: True if the path is not in ignored rules. False otherwise. + """ + is_hidden = not path.name.startswith(".") + is_pycache = "__pycache__" not in path.name + return is_hidden and is_pycache def ask_for_files(db_input) -> dict[str, str]: use_last_string = "" - selection_number = 1 + selection_number = 0 + is_valid_selection = False + can_use_last = False if "file_list.txt" in db_input: can_use_last = True use_last_string = ( - f"3 - Use previous file list (available at {db_input.path / 'file_list.txt'})" + f"2. Use previous file list (available at {os.path.join(db_input.path, 'file_list.txt')})\n" ) - selectionstr = f""" -How do you want to select the files? -1 - Use terminal -2 - Use a GUI -{use_last_string} + selection_str = f"""How do you want to select the files? -Select the option and press enter (default=1) : """ +0. Use Command-Line. +1. Use File explorer. +{use_last_string if len(use_last_string) > 1 else ""} +Select option and press Enter (default={selection_number}): """ file_path_list = [] - selected_number_str = input(selectionstr) + selected_number_str = input(selection_str) if selected_number_str: - selection_number = int(selected_number_str) - is_valid_selection = False - if selection_number == 1: + try: + selection_number = int(selected_number_str) + except ValueError: + print("Invalid number. Select a number from the list above.\n") + sys.exit(1) + if selection_number == 0: # Open terminal selection file_path_list = terminal_file_selector() is_valid_selection = True - elif selection_number == 2: + elif selection_number == 1: # Open GUI selection file_path_list = gui_file_selector() is_valid_selection = True else: - if can_use_last: - if selection_number == 3: - # Use previous file list - is_valid_selection = True + if can_use_last and selection_number == 2: + # Use previous file list + is_valid_selection = True if not is_valid_selection: - print("Invalid number. Select a number from the list above.") - return {} + print("Invalid number. Select a number from the list above.\n") + sys.exit(1) file_list_string = "" file_path_info = {} - if not selection_number == 3: + if not selection_number == 2: # New files for file_path in file_path_list: - file_list_string += file_path + "\n" + file_list_string += str(file_path) + "\n" # Return a dict with key=file_name and value=file_path file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path # Write in file_list so the user can edit and remember what was done db_input["file_list.txt"] = file_list_string else: - # If using the the previous file list, we dont need to write file_list.txt + # If using the the previous file list, we don't need to write file_list.txt file_list_string = db_input["file_list.txt"] for file_path in file_path_list: # Return a dict with key=file_name and value=file_path diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..26d18f6ec3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +dataclasses_json==0.5.9 +openai==0.27.8 +pytest==7.3.1 +termcolor==2.3.0 +typer==0.9.0 From 5615abc5841f90cfeb750420c5384a3b76ce9db1 Mon Sep 17 00:00:00 2001 From: Lucas Klein <59050136+lectair@users.noreply.github.com> Date: Wed, 19 Jul 2023 19:59:08 +0200 Subject: [PATCH 13/41] Delete requirements.txt --- requirements.txt | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 26d18f6ec3..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -dataclasses_json==0.5.9 -openai==0.27.8 -pytest==7.3.1 -termcolor==2.3.0 -typer==0.9.0 From ef689b6729d6b12afaec8cc6632b05c75b8b6a30 Mon Sep 17 00:00:00 2001 From: Lucas Klein <59050136+lectair@users.noreply.github.com> Date: Wed, 19 Jul 2023 19:59:21 +0200 Subject: [PATCH 14/41] Delete file_list.txt --- gpt_engineer/.gpteng/file_list.txt | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 gpt_engineer/.gpteng/file_list.txt diff --git a/gpt_engineer/.gpteng/file_list.txt b/gpt_engineer/.gpteng/file_list.txt deleted file mode 100644 index 78319baab1..0000000000 --- a/gpt_engineer/.gpteng/file_list.txt +++ /dev/null @@ -1,20 +0,0 @@ -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\__init__.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\ai.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\chat_to_files.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\collect.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\db.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\domain.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\file_selector.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\learning.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\main.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\fix_code -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\generate -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\implement_on_existing -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\philosophy -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\qa -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\respec -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\spec -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\unit_tests -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\use_feedback -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\use_qa -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\steps.py From 24256101b62e13d5d95c32f2e6214b4a6eebac9b Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Thu, 20 Jul 2023 23:16:56 -0300 Subject: [PATCH 15/41] fix typing for max and min value in range --- gpt_engineer/file_selector.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py index 58d3811a13..357b8f9094 100644 --- a/gpt_engineer/file_selector.py +++ b/gpt_engineer/file_selector.py @@ -126,7 +126,6 @@ def displayable(self) -> str: return "".join(reversed(parts)) - class TerminalFileSelector: def __init__(self, root_folder_path: Path) -> None: self.number_of_selectable_items = 0 @@ -174,23 +173,27 @@ def ask_for_selection(self) -> List[str]: List[str]: list of selected paths """ user_input = input( - "\nSelect files by entering the numbers separated by commas/spaces or specify range with a dash.\nExample: 1,2,3-5,7,9,13-15,18,20 (enter 'all' to select everything)\n\nSelect files: " + "\nSelect files by entering the numbers separated by commas/spaces or " + + "specify range with a dash. " + + "Example: 1,2,3-5,7,9,13-15,18,20 (enter 'all' to select everything)" + + "\n\nSelect files:" ) selected_paths = [] regex = r"\d+(-\d+)?([, ]\d+(-\d+)?)*" - if user_input.lower() == "all": selected_paths = self.file_path_list elif re.match(regex, user_input): try: - user_input = user_input.replace("", ",") if " " in user_input else user_input + user_input = ( + user_input.replace("", ",") if " " in user_input else user_input + ) selected_files = user_input.split(",") for file_number_str in selected_files: if "-" in file_number_str: - start, end = file_number_str.split("-") - start = int(start) - end = int(end) + start_str, end_str = file_number_str.split("-") + start = int(start_str) + end = int(end_str) for num in range(start, end + 1): selected_paths.append(str(self.selectable_file_paths[num])) else: @@ -235,7 +238,8 @@ def ask_for_files(db_input) -> dict[str, str]: if "file_list.txt" in db_input: can_use_last = True use_last_string = ( - f"2. Use previous file list (available at {os.path.join(db_input.path, 'file_list.txt')})\n" + "2. Use previous file list (available at " + + f"{os.path.join(db_input.path, 'file_list.txt')})\n" ) selection_str = f"""How do you want to select the files? From 64b4327e94122bd728cc9674842436d015275890 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 19:24:05 -0300 Subject: [PATCH 16/41] Add strategy to fix existing code in computer --- gpt_engineer/chat_to_files.py | 22 ++++++++++++++++- gpt_engineer/preprompts/implement_on_existing | 20 ++++++++++++++++ gpt_engineer/steps.py | 24 ++++++++++++++++++- 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 gpt_engineer/preprompts/implement_on_existing diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 6a3c781175..c40b1d6123 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,5 +1,5 @@ import re - +import os def parse_chat(chat): # -> List[Tuple[str, str]]: # Get all ``` blocks and preceding filenames @@ -40,3 +40,23 @@ def to_files(chat, workspace): files = parse_chat(chat) for file_name, file_content in files: workspace[file_name] = file_content + +def getCodeStrings(input): + filesPaths = input["file_list.txt"].split('\n') + filesDict = {} + for filePath in filesPaths: + with open(filePath, 'r') as file: + fileData = file.read() + fileName = os.path.basename(filePath).split('/')[-1] + filesDict[fileName] = fileData + print(filesDict) + return filesDict + +def formatFileToInput(fileName, fileContent): + filestr = f''' + {fileName} + ``` + {fileContent} + ``` + ''' + return filestr \ No newline at end of file diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing new file mode 100644 index 0000000000..613134ed21 --- /dev/null +++ b/gpt_engineer/preprompts/implement_on_existing @@ -0,0 +1,20 @@ +You are a expert developer and you are tasked to work in fixing an issue or creating a new functionaliy in a set of existing codebase. +After reading the implementation request, you will recieve the code one by one to you. +Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconections between each file, function and classes. +Think step by step and reason yourself to the right decisions to make sure we get it right. +The files are organized in the following way: + +FILENAME is the lowercase file name including the file extension, +LANG is the markup code block language for the code's language, and CODE is the code: + +FILENAME +``` +CODE +``` + +After that, implement the requested functionality by using the recieved code as base. +The received code not necessarely will be a standalone implementation, it could be part of a bigger project which you might not have all files. Trust that everything you need to know inside the received files. +Follow the same coding language and framework observed in the received code, using appropriate best practice file naming convention. +Make sure that code in different files are compatible with each other. +Ensure to implement the requested modification, if you are unsure, write a plausible implementation. +Before you finish, double check that all parts of the architecture is present in the files. \ No newline at end of file diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 1401e27bba..43679599e9 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -9,7 +9,7 @@ from termcolor import colored from gpt_engineer.ai import AI -from gpt_engineer.chat_to_files import to_files +from gpt_engineer.chat_to_files import to_files, getCodeStrings, formatFileToInput from gpt_engineer.db import DBs from gpt_engineer.learning import human_input @@ -21,6 +21,11 @@ def setup_sys_prompt(dbs: DBs) -> str: dbs.preprompts["generate"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] ) +def setup_sys_prompt_existing_code(dbs: DBs) -> str: + return ( + dbs.preprompts["implement_on_existing"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] + ) + def get_prompt(dbs: DBs) -> str: """While we migrate we have this fallback getter""" @@ -254,6 +259,21 @@ def use_feedback(ai: AI, dbs: DBs): to_files(messages[-1].content.strip(), dbs.workspace) return messages +def improve_existing_code(ai: AI, dbs: DBs): + filesInfo = getCodeStrings(dbs.input) + messages = [ + ai.fsystem(setup_sys_prompt_existing_code(dbs)), + ai.fuser(f"Instructions: {dbs.input['prompt']}") + ] + # Add files as input + for filename, filestr in filesInfo.items(): + codeInput = formatFileToInput(filename, filestr) + messages.append(ai.fuser(f"{codeInput}")) + + messages = ai.next(messages) + to_files(messages[-1]["content"], dbs.workspace) + return messages + def fix_code(ai: AI, dbs: DBs): messages = AI.deserialize_messages(dbs.logs[gen_code.__name__]) @@ -288,6 +308,7 @@ class Config(str, Enum): EXECUTE_ONLY = "execute_only" EVALUATE = "evaluate" USE_FEEDBACK = "use_feedback" + IMPROVE_CODE = "improve_code" # Different configs of what steps to run @@ -338,6 +359,7 @@ class Config(str, Enum): Config.USE_FEEDBACK: [use_feedback, gen_entrypoint, execute_entrypoint, human_review], Config.EXECUTE_ONLY: [execute_entrypoint], Config.EVALUATE: [execute_entrypoint, human_review], + Config.IMPROVE_CODE: [improve_existing_code] } # Future steps that can be added: From ca5b80641c940885dee2a77baa453cc91c68babe Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 19:31:44 -0300 Subject: [PATCH 17/41] add comments and remove debug prints --- gpt_engineer/chat_to_files.py | 3 ++- gpt_engineer/steps.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index c40b1d6123..e86bae8591 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -41,6 +41,7 @@ def to_files(chat, workspace): for file_name, file_content in files: workspace[file_name] = file_content +# Get code content from a code list def getCodeStrings(input): filesPaths = input["file_list.txt"].split('\n') filesDict = {} @@ -49,9 +50,9 @@ def getCodeStrings(input): fileData = file.read() fileName = os.path.basename(filePath).split('/')[-1] filesDict[fileName] = fileData - print(filesDict) return filesDict +# Format file for inputing to chat prompt def formatFileToInput(fileName, fileContent): filestr = f''' {fileName} diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 43679599e9..83dfb06fea 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -22,6 +22,9 @@ def setup_sys_prompt(dbs: DBs) -> str: ) def setup_sys_prompt_existing_code(dbs: DBs) -> str: + """ + Similar to code generation, but using an existing code base. + """ return ( dbs.preprompts["implement_on_existing"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] ) @@ -260,6 +263,12 @@ def use_feedback(ai: AI, dbs: DBs): return messages def improve_existing_code(ai: AI, dbs: DBs): + """ + Based on a list of existing files, ask the AI agent to improve, fix or add a new functionality + Necessary to have a 'file_list.txt' and a 'prompt' in the project folder. + The file_list.txt should have the path of the code to be changed + The prompt should have the request for change. + """ filesInfo = getCodeStrings(dbs.input) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), From c878b0f3ae2bb0aa809c855cf20920c683b43921 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 19:52:56 -0300 Subject: [PATCH 18/41] Format using black --- gpt_engineer/chat_to_files.py | 15 +++++++++------ gpt_engineer/steps.py | 10 +++++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index e86bae8591..29e8ace0e4 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,6 +1,7 @@ import re import os + def parse_chat(chat): # -> List[Tuple[str, str]]: # Get all ``` blocks and preceding filenames regex = r"(\S+)\n\s*```[^\n]*\n(.+?)```" @@ -41,23 +42,25 @@ def to_files(chat, workspace): for file_name, file_content in files: workspace[file_name] = file_content + # Get code content from a code list def getCodeStrings(input): - filesPaths = input["file_list.txt"].split('\n') + filesPaths = input["file_list.txt"].split("\n") filesDict = {} for filePath in filesPaths: - with open(filePath, 'r') as file: + with open(filePath, "r") as file: fileData = file.read() - fileName = os.path.basename(filePath).split('/')[-1] + fileName = os.path.basename(filePath).split("/")[-1] filesDict[fileName] = fileData return filesDict + # Format file for inputing to chat prompt def formatFileToInput(fileName, fileContent): - filestr = f''' + filestr = f""" {fileName} ``` {fileContent} ``` - ''' - return filestr \ No newline at end of file + """ + return filestr diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 83dfb06fea..03745e396e 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -21,12 +21,15 @@ def setup_sys_prompt(dbs: DBs) -> str: dbs.preprompts["generate"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] ) + def setup_sys_prompt_existing_code(dbs: DBs) -> str: """ Similar to code generation, but using an existing code base. """ return ( - dbs.preprompts["implement_on_existing"] + "\nUseful to know:\n" + dbs.preprompts["philosophy"] + dbs.preprompts["implement_on_existing"] + + "\nUseful to know:\n" + + dbs.preprompts["philosophy"] ) @@ -262,6 +265,7 @@ def use_feedback(ai: AI, dbs: DBs): to_files(messages[-1].content.strip(), dbs.workspace) return messages + def improve_existing_code(ai: AI, dbs: DBs): """ Based on a list of existing files, ask the AI agent to improve, fix or add a new functionality @@ -272,7 +276,7 @@ def improve_existing_code(ai: AI, dbs: DBs): filesInfo = getCodeStrings(dbs.input) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), - ai.fuser(f"Instructions: {dbs.input['prompt']}") + ai.fuser(f"Instructions: {dbs.input['prompt']}"), ] # Add files as input for filename, filestr in filesInfo.items(): @@ -368,7 +372,7 @@ class Config(str, Enum): Config.USE_FEEDBACK: [use_feedback, gen_entrypoint, execute_entrypoint, human_review], Config.EXECUTE_ONLY: [execute_entrypoint], Config.EVALUATE: [execute_entrypoint, human_review], - Config.IMPROVE_CODE: [improve_existing_code] + Config.IMPROVE_CODE: [improve_existing_code], } # Future steps that can be added: From 6aea25de150cdaa2348f2e3869a993ad7d2e3fb0 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 20:04:14 -0300 Subject: [PATCH 19/41] lint with ruff --- gpt_engineer/chat_to_files.py | 2 +- gpt_engineer/steps.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 29e8ace0e4..50f4ac9858 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,5 +1,5 @@ -import re import os +import re def parse_chat(chat): # -> List[Tuple[str, str]]: diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 03745e396e..209037da2e 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -9,7 +9,7 @@ from termcolor import colored from gpt_engineer.ai import AI -from gpt_engineer.chat_to_files import to_files, getCodeStrings, formatFileToInput +from gpt_engineer.chat_to_files import formatFileToInput, getCodeStrings, to_files from gpt_engineer.db import DBs from gpt_engineer.learning import human_input @@ -268,7 +268,8 @@ def use_feedback(ai: AI, dbs: DBs): def improve_existing_code(ai: AI, dbs: DBs): """ - Based on a list of existing files, ask the AI agent to improve, fix or add a new functionality + Based on a list of existing files, ask the AI agent to + improve, fix or add a new functionality Necessary to have a 'file_list.txt' and a 'prompt' in the project folder. The file_list.txt should have the path of the code to be changed The prompt should have the request for change. From f80ff8ee99cbd93e73102713a3a1a329adb57238 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sat, 1 Jul 2023 20:09:10 -0300 Subject: [PATCH 20/41] add new line at end of file --- gpt_engineer/preprompts/implement_on_existing | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing index 613134ed21..cf572517fe 100644 --- a/gpt_engineer/preprompts/implement_on_existing +++ b/gpt_engineer/preprompts/implement_on_existing @@ -1,5 +1,5 @@ You are a expert developer and you are tasked to work in fixing an issue or creating a new functionaliy in a set of existing codebase. -After reading the implementation request, you will recieve the code one by one to you. +After reading the implementation request, you will recieve the code one by one to you. Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconections between each file, function and classes. Think step by step and reason yourself to the right decisions to make sure we get it right. The files are organized in the following way: @@ -17,4 +17,4 @@ The received code not necessarely will be a standalone implementation, it could Follow the same coding language and framework observed in the received code, using appropriate best practice file naming convention. Make sure that code in different files are compatible with each other. Ensure to implement the requested modification, if you are unsure, write a plausible implementation. -Before you finish, double check that all parts of the architecture is present in the files. \ No newline at end of file +Before you finish, double check that all parts of the architecture is present in the files. From 4e4a9e25b031ed2cfe7295cc9186ccf1bc3175aa Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sun, 2 Jul 2023 20:26:19 -0300 Subject: [PATCH 21/41] Fix method naming convention and other reviewing details --- gpt_engineer/chat_to_files.py | 4 ++-- gpt_engineer/preprompts/implement_on_existing | 3 +-- gpt_engineer/steps.py | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 50f4ac9858..6129f76a2e 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -44,7 +44,7 @@ def to_files(chat, workspace): # Get code content from a code list -def getCodeStrings(input): +def get_code_strings(input): filesPaths = input["file_list.txt"].split("\n") filesDict = {} for filePath in filesPaths: @@ -56,7 +56,7 @@ def getCodeStrings(input): # Format file for inputing to chat prompt -def formatFileToInput(fileName, fileContent): +def format_file_to_input(fileName, fileContent): filestr = f""" {fileName} ``` diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing index cf572517fe..c0a150a2cb 100644 --- a/gpt_engineer/preprompts/implement_on_existing +++ b/gpt_engineer/preprompts/implement_on_existing @@ -4,8 +4,7 @@ Based on the request, you need to comprehend what needs to be modified by unders Think step by step and reason yourself to the right decisions to make sure we get it right. The files are organized in the following way: -FILENAME is the lowercase file name including the file extension, -LANG is the markup code block language for the code's language, and CODE is the code: +FILENAME is the lowercase file name including the file extension and CODE is the code: FILENAME ``` diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 209037da2e..df27fd9e47 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -9,7 +9,7 @@ from termcolor import colored from gpt_engineer.ai import AI -from gpt_engineer.chat_to_files import formatFileToInput, getCodeStrings, to_files +from gpt_engineer.chat_to_files import format_file_to_input, get_code_strings, to_files from gpt_engineer.db import DBs from gpt_engineer.learning import human_input @@ -274,14 +274,14 @@ def improve_existing_code(ai: AI, dbs: DBs): The file_list.txt should have the path of the code to be changed The prompt should have the request for change. """ - filesInfo = getCodeStrings(dbs.input) + filesInfo = get_code_strings(dbs.input) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), ai.fuser(f"Instructions: {dbs.input['prompt']}"), ] # Add files as input for filename, filestr in filesInfo.items(): - codeInput = formatFileToInput(filename, filestr) + codeInput = format_file_to_input(filename, filestr) messages.append(ai.fuser(f"{codeInput}")) messages = ai.next(messages) From 78fcde15ff4d45fbab829348ab68a751f9880f30 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Mon, 3 Jul 2023 19:16:37 -0300 Subject: [PATCH 22/41] Fix identation and typos --- gpt_engineer/chat_to_files.py | 2 ++ gpt_engineer/preprompts/implement_on_existing | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 6129f76a2e..766882248c 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -48,8 +48,10 @@ def get_code_strings(input): filesPaths = input["file_list.txt"].split("\n") filesDict = {} for filePath in filesPaths: + fileData = "" with open(filePath, "r") as file: fileData = file.read() + if fileData: fileName = os.path.basename(filePath).split("/")[-1] filesDict[fileName] = fileData return filesDict diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing index c0a150a2cb..5f69ed3e2a 100644 --- a/gpt_engineer/preprompts/implement_on_existing +++ b/gpt_engineer/preprompts/implement_on_existing @@ -1,5 +1,5 @@ You are a expert developer and you are tasked to work in fixing an issue or creating a new functionaliy in a set of existing codebase. -After reading the implementation request, you will recieve the code one by one to you. +After reading the implementation request, you will receive the code one by one to you. Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconections between each file, function and classes. Think step by step and reason yourself to the right decisions to make sure we get it right. The files are organized in the following way: @@ -11,7 +11,7 @@ FILENAME CODE ``` -After that, implement the requested functionality by using the recieved code as base. +After that, implement the requested functionality by using the received code as base. The received code not necessarely will be a standalone implementation, it could be part of a bigger project which you might not have all files. Trust that everything you need to know inside the received files. Follow the same coding language and framework observed in the received code, using appropriate best practice file naming convention. Make sure that code in different files are compatible with each other. From 8d390e3ded82642c552c37d766dc4d06e93403d5 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sun, 9 Jul 2023 00:43:35 -0300 Subject: [PATCH 23/41] Improve code with .gpteng folder, add file selection window with tkinter --- gpt_engineer/chat_to_files.py | 67 ++++++++++++++++++++++++++++++++--- gpt_engineer/main.py | 31 ++++++++++++++++ gpt_engineer/steps.py | 39 ++++++++++++++++---- 3 files changed, 126 insertions(+), 11 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 766882248c..a58953d74d 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,5 +1,7 @@ import os import re +import tkinter as tk +import tkinter.filedialog as fd def parse_chat(chat): # -> List[Tuple[str, str]]: @@ -43,9 +45,36 @@ def to_files(chat, workspace): workspace[file_name] = file_content -# Get code content from a code list -def get_code_strings(input): - filesPaths = input["file_list.txt"].split("\n") +def overwrite_files(chat, dbs, replace_files): + """ + Replace the AI files to the older local files. + """ + dbs.workspace["all_output.txt"] = chat + + files = parse_chat(chat) + for file_name, file_content in files: + # Verify if the file created by the AI agent was in the input list + if file_name in replace_files: + # If the AI created a file from our input list, we replace it. + with open(replace_files[file_name], "w") as text_file: + text_file.write(file_content) + else: + # If the AI create a new file I don't know where to put it yet + # maybe we can think in a smarter solution for this in the future + # like asking the AI where to put it. + # + # by now, just add this to the workspace inside .gpteng folder + print( + f"Could not find file path for '{file_name}', creating file in workspace" + ) + dbs.workspace[file_name] = file_content + + +def get_code_strings(input) -> dict[str, str]: + """ + Read file_list.txt and return file names and its content. + """ + filesPaths = input["file_list.txt"].strip().split("\n") filesDict = {} for filePath in filesPaths: fileData = "" @@ -57,8 +86,10 @@ def get_code_strings(input): return filesDict -# Format file for inputing to chat prompt -def format_file_to_input(fileName, fileContent): +def format_file_to_input(fileName: str, fileContent: str) -> str: + """ + Format a file string to use as input to AI agent + """ filestr = f""" {fileName} ``` @@ -66,3 +97,29 @@ def format_file_to_input(fileName, fileContent): ``` """ return filestr + + +def ask_for_files(input) -> dict[str, str]: + """ + Display a tkinter file selection window to select context files. + Return a dictionary of file_name as key and file_path as value + """ + root = tk.Tk() + root.withdraw() + root.call("wm", "attributes", ".", "-topmost", True) + file_list = list( + fd.askopenfilenames( + parent=root, + initialdir=os.getcwd(), + title="Select relevant files for your change:", + ) + ) + file_list_string = "" + file_path_info = {} + for file_path in file_list: + file_list_string += file_path + "\n" + # Return a dict with key=file_name and value=file_path + file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path + # Write in file_list so the user can edit and remember what was done + input["file_list.txt"] = file_list_string + return file_path_info diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index 72db1c508b..93409fff1a 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -1,5 +1,8 @@ import logging +import os +import shutil + from pathlib import Path import typer @@ -21,10 +24,38 @@ def main( steps_config: StepsConfig = typer.Option( StepsConfig.DEFAULT, "--steps", "-s", help="decide which steps to run" ), + improve_option: bool = typer.Option( + False, + "--improve", + "-i", + help="Improve code from existing project.", + ), verbose: bool = typer.Option(False, "--verbose", "-v"), ): logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO) + + input_path = Path(project_path).absolute() + + # For the improve option take current project as path and add .gpteng folder + # By now, ignoring the 'project_path' argument + if improve_option: + input_path = Path(os.getcwd()).absolute() / ".gpteng" + input_path.mkdir(parents=True, exist_ok=True) + # The default option for the --improve is the IMPROVE_CODE, not DEFAULT + # I know this looks ugly, not sure if it is the best way to do that... + # we can change that in the future. + if steps_config == steps.Config.DEFAULT: + steps_config = steps.Config.IMPROVE_CODE + + memory_path = input_path / f"{run_prefix}memory" + workspace_path = input_path / f"{run_prefix}workspace" + + if delete_existing: + # Delete files and subdirectories in paths + shutil.rmtree(memory_path, ignore_errors=True) + shutil.rmtree(workspace_path, ignore_errors=True) + model = fallback_model(model) ai = AI( model_name=model, diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index df27fd9e47..6b9e055210 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -9,7 +9,13 @@ from termcolor import colored from gpt_engineer.ai import AI -from gpt_engineer.chat_to_files import format_file_to_input, get_code_strings, to_files +from gpt_engineer.chat_to_files import ( + ask_for_files, + format_file_to_input, + get_code_strings, + overwrite_files, + to_files, +) from gpt_engineer.db import DBs from gpt_engineer.learning import human_input @@ -268,13 +274,33 @@ def use_feedback(ai: AI, dbs: DBs): def improve_existing_code(ai: AI, dbs: DBs): """ - Based on a list of existing files, ask the AI agent to + Ask the user for a list of paths, ask the AI agent to improve, fix or add a new functionality - Necessary to have a 'file_list.txt' and a 'prompt' in the project folder. - The file_list.txt should have the path of the code to be changed - The prompt should have the request for change. + A file selection will appear to select the files. + The terminal will ask for the prompt. """ + file_path_info = ask_for_files(dbs.input) filesInfo = get_code_strings(dbs.input) + dbs.input["prompt"] = input( + "\nWhat do you need to improve with the selected files?\n" + ) + + confirmstr = f""" +----------------------------- +The following files will be used in the improvement process: +{dbs.input["file_list.txt"]} + +The inserted prompt is the following: +'{dbs.input['prompt']}' +----------------------------- + +You can change these files in .gpteng folder ({dbs.input.path}) in your project +before proceeding. + +Press enter to proceed with modifications. + +""" + input(confirmstr) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), ai.fuser(f"Instructions: {dbs.input['prompt']}"), @@ -285,7 +311,8 @@ def improve_existing_code(ai: AI, dbs: DBs): messages.append(ai.fuser(f"{codeInput}")) messages = ai.next(messages) - to_files(messages[-1]["content"], dbs.workspace) + # Maybe we should add another step called "replace" or "overwrite" + overwrite_files(messages[-1]["content"], dbs, replace_files=file_path_info) return messages From 214fb7750b75f05e03ca502c970e944253fa2200 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Mon, 10 Jul 2023 23:26:28 -0300 Subject: [PATCH 24/41] add file selector using terminal --- gpt_engineer/chat_to_files.py | 28 ---- gpt_engineer/file_selector.py | 274 ++++++++++++++++++++++++++++++++++ gpt_engineer/steps.py | 2 +- 3 files changed, 275 insertions(+), 29 deletions(-) create mode 100644 gpt_engineer/file_selector.py diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index a58953d74d..09719faf13 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,7 +1,5 @@ import os import re -import tkinter as tk -import tkinter.filedialog as fd def parse_chat(chat): # -> List[Tuple[str, str]]: @@ -97,29 +95,3 @@ def format_file_to_input(fileName: str, fileContent: str) -> str: ``` """ return filestr - - -def ask_for_files(input) -> dict[str, str]: - """ - Display a tkinter file selection window to select context files. - Return a dictionary of file_name as key and file_path as value - """ - root = tk.Tk() - root.withdraw() - root.call("wm", "attributes", ".", "-topmost", True) - file_list = list( - fd.askopenfilenames( - parent=root, - initialdir=os.getcwd(), - title="Select relevant files for your change:", - ) - ) - file_list_string = "" - file_path_info = {} - for file_path in file_list: - file_list_string += file_path + "\n" - # Return a dict with key=file_name and value=file_path - file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path - # Write in file_list so the user can edit and remember what was done - input["file_list.txt"] = file_list_string - return file_path_info diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py new file mode 100644 index 0000000000..38a415dee2 --- /dev/null +++ b/gpt_engineer/file_selector.py @@ -0,0 +1,274 @@ +import os +import tkinter as tk +import tkinter.filedialog as fd + +from pathlib import Path +from typing import List, Union + + +class DisplayablePath(object): + """ + A class representing a displayable path in a file explorer. + """ + + display_filename_prefix_middle = "├── " + display_filename_prefix_last = "└── " + display_parent_prefix_middle = " " + display_parent_prefix_last = "│ " + + def __init__( + self, path: Union[str, Path], parent_path: "DisplayablePath", is_last: bool + ): + """ + Initialize a DisplayablePath object. + + Args: + path (Union[str, Path]): The path of the file or directory. + parent_path (DisplayablePath): The parent path of the file or directory. + is_last (bool): Whether the file or directory is the last child of its parent. + """ + self.depth: int = 0 + self.path = Path(str(path)) + self.parent = parent_path + self.is_last = is_last + if self.parent: + self.depth = self.parent.depth + 1 + + @property + def displayname(self) -> str: + """ + Get the display name of the file or directory. + + Returns: + str: The display name. + """ + if self.path.is_dir(): + return self.path.name + "/" + return self.path.name + + @classmethod + def make_tree(cls, root: Union[str, Path], parent=None, is_last=False, criteria=None): + """ + Generate a tree of DisplayablePath objects. + + Args: + root: The root path of the tree. + parent: The parent path of the root path. Defaults to None. + is_last: Whether the root path is the last child of its parent. + criteria: The criteria function to filter the paths. Defaults to None. + + Yields: + DisplayablePath: The DisplayablePath objects in the tree. + """ + root = Path(str(root)) + criteria = criteria or cls._default_criteria + + displayable_root = cls(root, parent, is_last) + yield displayable_root + + children = sorted( + list(path for path in root.iterdir() if criteria(path)), + key=lambda s: str(s).lower(), + ) + count = 1 + for path in children: + is_last = count == len(children) + if path.is_dir(): + yield from cls.make_tree( + path, parent=displayable_root, is_last=is_last, criteria=criteria + ) + else: + yield cls(path, displayable_root, is_last) + count += 1 + + @classmethod + def _default_criteria(cls, path: Path) -> bool: + """ + The default criteria function to filter the paths. + + Args: + path: The path to check. + + Returns: + bool: True if the path should be included, False otherwise. + """ + return True + + def displayable(self) -> str: + """ + Get the displayable string representation of the file or directory. + + Returns: + str: The displayable string representation. + """ + if self.parent is None: + return self.displayname + + _filename_prefix = ( + self.display_filename_prefix_last + if self.is_last + else self.display_filename_prefix_middle + ) + + parts = ["{!s} {!s}".format(_filename_prefix, self.displayname)] + + parent = self.parent + while parent and parent.parent is not None: + parts.append( + self.display_parent_prefix_middle + if parent.is_last + else self.display_parent_prefix_last + ) + parent = parent.parent + + return "".join(reversed(parts)) + + +class TerminalFileSelector: + def __init__(self, root_folder_path: Path) -> None: + self.number_of_selectable_items = 0 + self.selectable_file_paths: dict[int, str] = {} + self.db_paths = DisplayablePath.make_tree( + root_folder_path, parent=None, criteria=self.is_in_ignoring_extentions + ) + + def display(self): + """ + Select files from a directory and display the selected files. + """ + count = 1 + file_path_enumeration = {} + for path in self.db_paths: + n_digits = len(str(count)) + n_spaces = 3 - n_digits + if n_spaces < 0: + # We can only print 1000 aligned files. I think it is decent enough + n_spaces = 0 + spaces_str = " " * n_spaces + if not path.path.is_dir(): + print(f"{count}. {spaces_str}{path.displayable()}") + file_path_enumeration[count] = path.path + count += 1 + else: + # By now we do not accept selecting entire dirs. + # But could add that in the future. Just need to add more functions + # and remove this else block... + number_space = " " * n_digits + print(f"{number_space} {spaces_str}{path.displayable()}") + + self.number_of_selectable_items = count + self.selectable_file_paths = file_path_enumeration + + def ask_for_selection(self) -> List[str]: + user_input = input( + "Select files by entering the numbers separated by spaces or commas: " + ) + user_input = user_input.replace(",", " ") + selected_files = user_input.split() + selected_paths = [] + for file_number_str in selected_files: + try: + file_number = int(file_number_str) + if 1 <= file_number <= self.number_of_selectable_items: + selected_paths.append(str(self.selectable_file_paths[file_number])) + except ValueError: + pass + return selected_paths + + def is_in_ignoring_extentions(self, path: Path) -> bool: + """ + Check if a path is not hidden or in the __pycache__ directory. + + Args: + path: The path to check. + + Returns: + bool: True if the path is not in ignored rules. False otherwise. + """ + is_hidden = not path.name.startswith(".") + is_pycache = "__pycache__" not in path.name + return is_hidden and is_pycache + + +def ask_for_files(db_input) -> dict[str, str]: + use_last_string = "" + selection_number = 1 + if "file_list.txt" in db_input: + can_use_last = True + use_last_string = ( + f"3 - Use previous file list (available at {db_input.path / 'file_list.txt'})" + ) + selectionstr = f""" +How do you want to select the files? +1 - Use terminal +2 - Use a GUI +{use_last_string} + +Select the option and press enter (default=1) : """ + file_path_list = [] + selected_number_str = input(selectionstr) + if selected_number_str: + selection_number = int(selected_number_str) + is_valid_selection = False + if selection_number == 1: + # Open terminal selection + file_path_list = terminal_file_selector() + is_valid_selection = True + elif selection_number == 2: + # Open GUI selection + file_path_list = gui_file_selector() + is_valid_selection = True + else: + if can_use_last: + if selection_number == 3: + # Use previous file list + is_valid_selection = True + if not is_valid_selection: + print("Invalid number. Select a number from the list above.") + return {} + + file_list_string = "" + file_path_info = {} + if not selection_number == 3: + # New files + for file_path in file_path_list: + file_list_string += file_path + "\n" + # Return a dict with key=file_name and value=file_path + file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path + # Write in file_list so the user can edit and remember what was done + db_input["file_list.txt"] = file_list_string + else: + # If using the the previous file list, we dont need to write file_list.txt + file_list_string = db_input["file_list.txt"] + for file_path in file_path_list: + # Return a dict with key=file_name and value=file_path + file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path + + return file_path_info + + +def gui_file_selector() -> List[str]: + """ + Display a tkinter file selection window to select context files. + """ + root = tk.Tk() + root.withdraw() + root.call("wm", "attributes", ".", "-topmost", True) + file_list = list( + fd.askopenfilenames( + parent=root, + initialdir=os.getcwd(), + title="Select files to improve (or give context):", + ) + ) + return file_list + + +def terminal_file_selector() -> List[str]: + """ + Display a terminal file selection to select context files. + """ + file_selector = TerminalFileSelector(Path(os.getcwd())) + file_selector.display() + selected_list = file_selector.ask_for_selection() + return selected_list diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 6b9e055210..1575fece47 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -10,13 +10,13 @@ from gpt_engineer.ai import AI from gpt_engineer.chat_to_files import ( - ask_for_files, format_file_to_input, get_code_strings, overwrite_files, to_files, ) from gpt_engineer.db import DBs +from gpt_engineer.file_selector import ask_for_files from gpt_engineer.learning import human_input Message = Union[AIMessage, HumanMessage, SystemMessage] From 39599b6ca6e6a06b43618b76065c004809b083f2 Mon Sep 17 00:00:00 2001 From: Lucas Klein Date: Tue, 11 Jul 2023 09:14:46 +0200 Subject: [PATCH 25/41] Added better functionality to file selector input, fixed some bugs, misspellings and cleaned up. Applied leomariga suggestions. --- gpt_engineer/.gpteng/file_list.txt | 20 +++++ gpt_engineer/file_selector.py | 118 ++++++++++++++++++----------- requirements.txt | 5 ++ 3 files changed, 98 insertions(+), 45 deletions(-) create mode 100644 gpt_engineer/.gpteng/file_list.txt create mode 100644 requirements.txt diff --git a/gpt_engineer/.gpteng/file_list.txt b/gpt_engineer/.gpteng/file_list.txt new file mode 100644 index 0000000000..78319baab1 --- /dev/null +++ b/gpt_engineer/.gpteng/file_list.txt @@ -0,0 +1,20 @@ +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\__init__.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\ai.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\chat_to_files.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\collect.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\db.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\domain.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\file_selector.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\learning.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\main.py +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\fix_code +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\generate +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\implement_on_existing +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\philosophy +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\qa +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\respec +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\spec +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\unit_tests +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\use_feedback +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\use_qa +C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\steps.py diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py index 38a415dee2..4da8c1b65b 100644 --- a/gpt_engineer/file_selector.py +++ b/gpt_engineer/file_selector.py @@ -1,4 +1,6 @@ import os +import re +import sys import tkinter as tk import tkinter.filedialog as fd @@ -35,7 +37,7 @@ def __init__( self.depth = self.parent.depth + 1 @property - def displayname(self) -> str: + def display_name(self) -> str: """ Get the display name of the file or directory. @@ -102,7 +104,7 @@ def displayable(self) -> str: str: The displayable string representation. """ if self.parent is None: - return self.displayname + return self.display_name _filename_prefix = ( self.display_filename_prefix_last @@ -110,7 +112,7 @@ def displayable(self) -> str: else self.display_filename_prefix_middle ) - parts = ["{!s} {!s}".format(_filename_prefix, self.displayname)] + parts = ["{!s} {!s}".format(_filename_prefix, self.display_name)] parent = self.parent while parent and parent.parent is not None: @@ -124,20 +126,23 @@ def displayable(self) -> str: return "".join(reversed(parts)) + class TerminalFileSelector: def __init__(self, root_folder_path: Path) -> None: self.number_of_selectable_items = 0 self.selectable_file_paths: dict[int, str] = {} + self.file_path_list: list = [] self.db_paths = DisplayablePath.make_tree( - root_folder_path, parent=None, criteria=self.is_in_ignoring_extentions + root_folder_path, parent=None, criteria=self.is_in_ignoring_extensions ) def display(self): """ Select files from a directory and display the selected files. """ - count = 1 + count = 0 file_path_enumeration = {} + file_path_list = [] for path in self.db_paths: n_digits = len(str(count)) n_spaces = 3 - n_digits @@ -148,6 +153,7 @@ def display(self): if not path.path.is_dir(): print(f"{count}. {spaces_str}{path.displayable()}") file_path_enumeration[count] = path.path + file_path_list.append(path.path) count += 1 else: # By now we do not accept selecting entire dirs. @@ -157,88 +163,110 @@ def display(self): print(f"{number_space} {spaces_str}{path.displayable()}") self.number_of_selectable_items = count + self.file_path_list = file_path_list self.selectable_file_paths = file_path_enumeration def ask_for_selection(self) -> List[str]: user_input = input( - "Select files by entering the numbers separated by spaces or commas: " + "\nSelect files by entering the numbers separated by commas/spaces or specify range with a dash.\nExample: 1,2,3-5,7,9,13-15,18,20 (enter 'all' to select everything)\n\nSelect files: " ) - user_input = user_input.replace(",", " ") - selected_files = user_input.split() selected_paths = [] - for file_number_str in selected_files: + regex = r"\d+(-\d+)?([, ]\d+(-\d+)?)*" + + + if user_input.lower() == "all": + selected_paths = self.file_path_list + elif re.match(regex, user_input): try: - file_number = int(file_number_str) - if 1 <= file_number <= self.number_of_selectable_items: - selected_paths.append(str(self.selectable_file_paths[file_number])) + user_input = user_input.replace("", ",") if " " in user_input else user_input + selected_files = user_input.split(",") + for file_number_str in selected_files: + if "-" in file_number_str: + start, end = file_number_str.split("-") + start = int(start) + end = int(end) + for num in range(start, end + 1): + selected_paths.append(str(self.selectable_file_paths[num])) + else: + num = int(file_number_str) + selected_paths.append(str(self.selectable_file_paths[num])) + except ValueError: pass + else: + print("Please use a valid number/series of numbers.\n") + sys.exit(1) + return selected_paths - def is_in_ignoring_extentions(self, path: Path) -> bool: - """ - Check if a path is not hidden or in the __pycache__ directory. - Args: - path: The path to check. +def is_in_ignoring_extensions(path: Path) -> bool: + """ + Check if a path is not hidden or in the __pycache__ directory. - Returns: - bool: True if the path is not in ignored rules. False otherwise. - """ - is_hidden = not path.name.startswith(".") - is_pycache = "__pycache__" not in path.name - return is_hidden and is_pycache + Args: + path: The path to check. + + Returns: + bool: True if the path is not in ignored rules. False otherwise. + """ + is_hidden = not path.name.startswith(".") + is_pycache = "__pycache__" not in path.name + return is_hidden and is_pycache def ask_for_files(db_input) -> dict[str, str]: use_last_string = "" - selection_number = 1 + selection_number = 0 + is_valid_selection = False + can_use_last = False if "file_list.txt" in db_input: can_use_last = True use_last_string = ( - f"3 - Use previous file list (available at {db_input.path / 'file_list.txt'})" + f"2. Use previous file list (available at {os.path.join(db_input.path, 'file_list.txt')})\n" ) - selectionstr = f""" -How do you want to select the files? -1 - Use terminal -2 - Use a GUI -{use_last_string} + selection_str = f"""How do you want to select the files? -Select the option and press enter (default=1) : """ +0. Use Command-Line. +1. Use File explorer. +{use_last_string if len(use_last_string) > 1 else ""} +Select option and press Enter (default={selection_number}): """ file_path_list = [] - selected_number_str = input(selectionstr) + selected_number_str = input(selection_str) if selected_number_str: - selection_number = int(selected_number_str) - is_valid_selection = False - if selection_number == 1: + try: + selection_number = int(selected_number_str) + except ValueError: + print("Invalid number. Select a number from the list above.\n") + sys.exit(1) + if selection_number == 0: # Open terminal selection file_path_list = terminal_file_selector() is_valid_selection = True - elif selection_number == 2: + elif selection_number == 1: # Open GUI selection file_path_list = gui_file_selector() is_valid_selection = True else: - if can_use_last: - if selection_number == 3: - # Use previous file list - is_valid_selection = True + if can_use_last and selection_number == 2: + # Use previous file list + is_valid_selection = True if not is_valid_selection: - print("Invalid number. Select a number from the list above.") - return {} + print("Invalid number. Select a number from the list above.\n") + sys.exit(1) file_list_string = "" file_path_info = {} - if not selection_number == 3: + if not selection_number == 2: # New files for file_path in file_path_list: - file_list_string += file_path + "\n" + file_list_string += str(file_path) + "\n" # Return a dict with key=file_name and value=file_path file_path_info[os.path.basename(file_path).split("/")[-1]] = file_path # Write in file_list so the user can edit and remember what was done db_input["file_list.txt"] = file_list_string else: - # If using the the previous file list, we dont need to write file_list.txt + # If using the the previous file list, we don't need to write file_list.txt file_list_string = db_input["file_list.txt"] for file_path in file_path_list: # Return a dict with key=file_name and value=file_path diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..26d18f6ec3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +dataclasses_json==0.5.9 +openai==0.27.8 +pytest==7.3.1 +termcolor==2.3.0 +typer==0.9.0 From 8ad46a34c11f4b60c18f6e10dcaa23e8365bf2a6 Mon Sep 17 00:00:00 2001 From: Lucas Klein <59050136+lectair@users.noreply.github.com> Date: Wed, 19 Jul 2023 19:59:08 +0200 Subject: [PATCH 26/41] Delete requirements.txt --- requirements.txt | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 26d18f6ec3..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -dataclasses_json==0.5.9 -openai==0.27.8 -pytest==7.3.1 -termcolor==2.3.0 -typer==0.9.0 From 0f9bbea7ee09b9ac28b0a875da00c9728cfd76c3 Mon Sep 17 00:00:00 2001 From: Lucas Klein <59050136+lectair@users.noreply.github.com> Date: Wed, 19 Jul 2023 19:59:21 +0200 Subject: [PATCH 27/41] Delete file_list.txt --- gpt_engineer/.gpteng/file_list.txt | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 gpt_engineer/.gpteng/file_list.txt diff --git a/gpt_engineer/.gpteng/file_list.txt b/gpt_engineer/.gpteng/file_list.txt deleted file mode 100644 index 78319baab1..0000000000 --- a/gpt_engineer/.gpteng/file_list.txt +++ /dev/null @@ -1,20 +0,0 @@ -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\__init__.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\ai.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\chat_to_files.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\collect.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\db.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\domain.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\file_selector.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\learning.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\main.py -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\fix_code -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\generate -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\implement_on_existing -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\philosophy -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\qa -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\respec -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\spec -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\unit_tests -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\use_feedback -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\preprompts\use_qa -C:\Users\Lucas\PycharmProjects\gpt-engineer\gpt_engineer\steps.py From d46c3f6e467d926695a79c8dfee873a9c918b013 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Sun, 16 Jul 2023 15:34:53 -0300 Subject: [PATCH 28/41] fix typos and pep8 case rules, change workspace to cwd --- gpt_engineer/chat_to_files.py | 34 +++++++++---------- gpt_engineer/file_selector.py | 14 +++++++- gpt_engineer/main.py | 7 ++-- gpt_engineer/preprompts/implement_on_existing | 6 ++-- gpt_engineer/steps.py | 12 +++---- 5 files changed, 43 insertions(+), 30 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 09719faf13..c608a9bb24 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -72,26 +72,26 @@ def get_code_strings(input) -> dict[str, str]: """ Read file_list.txt and return file names and its content. """ - filesPaths = input["file_list.txt"].strip().split("\n") - filesDict = {} - for filePath in filesPaths: - fileData = "" - with open(filePath, "r") as file: - fileData = file.read() - if fileData: - fileName = os.path.basename(filePath).split("/")[-1] - filesDict[fileName] = fileData - return filesDict - - -def format_file_to_input(fileName: str, fileContent: str) -> str: + files_paths = input["file_list.txt"].strip().split("\n") + files_dict = {} + for file_path in files_paths: + file_data = "" + with open(file_path, "r") as file: + file_data = file.read() + if file_data: + file_name = os.path.basename(file_path).split("/")[-1] + files_dict[file_name] = file_data + return files_dict + + +def format_file_to_input(file_name: str, file_content: str) -> str: """ Format a file string to use as input to AI agent """ - filestr = f""" - {fileName} + file_str = f""" + {file_name} ``` - {fileContent} + {file_content} ``` """ - return filestr + return file_str diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py index 4da8c1b65b..2870ad1ad3 100644 --- a/gpt_engineer/file_selector.py +++ b/gpt_engineer/file_selector.py @@ -126,7 +126,6 @@ def displayable(self) -> str: return "".join(reversed(parts)) - class TerminalFileSelector: def __init__(self, root_folder_path: Path) -> None: self.number_of_selectable_items = 0 @@ -167,6 +166,12 @@ def display(self): self.selectable_file_paths = file_path_enumeration def ask_for_selection(self) -> List[str]: + """ + Ask user to select files from the terminal after displaying it + + Returns: + List[str]: list of selected paths + """ user_input = input( "\nSelect files by entering the numbers separated by commas/spaces or specify range with a dash.\nExample: 1,2,3-5,7,9,13-15,18,20 (enter 'all' to select everything)\n\nSelect files: " ) @@ -216,6 +221,13 @@ def is_in_ignoring_extensions(path: Path) -> bool: def ask_for_files(db_input) -> dict[str, str]: + """ + Ask user to select files to improve. + It can be done by terminal, gui, or using the old selection. + + Returns: + dict[str, str]: Dictionary where key = file name and value = file path + """ use_last_string = "" selection_number = 0 is_valid_selection = False diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index 93409fff1a..f9585e9c6f 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -37,6 +37,8 @@ def main( input_path = Path(project_path).absolute() + memory_path = input_path / f"{run_prefix}memory" + workspace_path = input_path / f"{run_prefix}workspace" # For the improve option take current project as path and add .gpteng folder # By now, ignoring the 'project_path' argument if improve_option: @@ -47,9 +49,8 @@ def main( # we can change that in the future. if steps_config == steps.Config.DEFAULT: steps_config = steps.Config.IMPROVE_CODE - - memory_path = input_path / f"{run_prefix}memory" - workspace_path = input_path / f"{run_prefix}workspace" + memory_path = input_path / f"{run_prefix}memory" + workspace_path = Path(os.getcwd()).absolute() if delete_existing: # Delete files and subdirectories in paths diff --git a/gpt_engineer/preprompts/implement_on_existing b/gpt_engineer/preprompts/implement_on_existing index 5f69ed3e2a..7f1fafc96e 100644 --- a/gpt_engineer/preprompts/implement_on_existing +++ b/gpt_engineer/preprompts/implement_on_existing @@ -1,6 +1,6 @@ -You are a expert developer and you are tasked to work in fixing an issue or creating a new functionaliy in a set of existing codebase. +You are a expert developer and you are tasked to work in fixing an issue or creating a new functionality in a set of existing codebase. After reading the implementation request, you will receive the code one by one to you. -Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconections between each file, function and classes. +Based on the request, you need to comprehend what needs to be modified by understanding the implementation of the received code and the interconnections between each file, function and classes. Think step by step and reason yourself to the right decisions to make sure we get it right. The files are organized in the following way: @@ -12,7 +12,7 @@ CODE ``` After that, implement the requested functionality by using the received code as base. -The received code not necessarely will be a standalone implementation, it could be part of a bigger project which you might not have all files. Trust that everything you need to know inside the received files. +The received code not necessarily will be a standalone implementation, it could be part of a bigger project which you might not have all files. Trust that everything you need to know inside the received files. Follow the same coding language and framework observed in the received code, using appropriate best practice file naming convention. Make sure that code in different files are compatible with each other. Ensure to implement the requested modification, if you are unsure, write a plausible implementation. diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 1575fece47..dace53f121 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -280,12 +280,12 @@ def improve_existing_code(ai: AI, dbs: DBs): The terminal will ask for the prompt. """ file_path_info = ask_for_files(dbs.input) - filesInfo = get_code_strings(dbs.input) + files_info = get_code_strings(dbs.input) dbs.input["prompt"] = input( "\nWhat do you need to improve with the selected files?\n" ) - confirmstr = f""" + confirm_str = f""" ----------------------------- The following files will be used in the improvement process: {dbs.input["file_list.txt"]} @@ -300,15 +300,15 @@ def improve_existing_code(ai: AI, dbs: DBs): Press enter to proceed with modifications. """ - input(confirmstr) + input(confirm_str) messages = [ ai.fsystem(setup_sys_prompt_existing_code(dbs)), ai.fuser(f"Instructions: {dbs.input['prompt']}"), ] # Add files as input - for filename, filestr in filesInfo.items(): - codeInput = format_file_to_input(filename, filestr) - messages.append(ai.fuser(f"{codeInput}")) + for file_name, file_str in files_info.items(): + code_input = format_file_to_input(file_name, file_str) + messages.append(ai.fuser(f"{code_input}")) messages = ai.next(messages) # Maybe we should add another step called "replace" or "overwrite" From eead934855864e4be4967787c2a4331176ea61b3 Mon Sep 17 00:00:00 2001 From: Leonardo Mariga Date: Thu, 20 Jul 2023 23:16:56 -0300 Subject: [PATCH 29/41] fix typing for max and min value in range --- gpt_engineer/file_selector.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py index 2870ad1ad3..c93dbdec58 100644 --- a/gpt_engineer/file_selector.py +++ b/gpt_engineer/file_selector.py @@ -173,23 +173,27 @@ def ask_for_selection(self) -> List[str]: List[str]: list of selected paths """ user_input = input( - "\nSelect files by entering the numbers separated by commas/spaces or specify range with a dash.\nExample: 1,2,3-5,7,9,13-15,18,20 (enter 'all' to select everything)\n\nSelect files: " + "\nSelect files by entering the numbers separated by commas/spaces or " + + "specify range with a dash. " + + "Example: 1,2,3-5,7,9,13-15,18,20 (enter 'all' to select everything)" + + "\n\nSelect files:" ) selected_paths = [] regex = r"\d+(-\d+)?([, ]\d+(-\d+)?)*" - if user_input.lower() == "all": selected_paths = self.file_path_list elif re.match(regex, user_input): try: - user_input = user_input.replace("", ",") if " " in user_input else user_input + user_input = ( + user_input.replace("", ",") if " " in user_input else user_input + ) selected_files = user_input.split(",") for file_number_str in selected_files: if "-" in file_number_str: - start, end = file_number_str.split("-") - start = int(start) - end = int(end) + start_str, end_str = file_number_str.split("-") + start = int(start_str) + end = int(end_str) for num in range(start, end + 1): selected_paths.append(str(self.selectable_file_paths[num])) else: @@ -235,7 +239,8 @@ def ask_for_files(db_input) -> dict[str, str]: if "file_list.txt" in db_input: can_use_last = True use_last_string = ( - f"2. Use previous file list (available at {os.path.join(db_input.path, 'file_list.txt')})\n" + "2. Use previous file list (available at " + + f"{os.path.join(db_input.path, 'file_list.txt')})\n" ) selection_str = f"""How do you want to select the files? From 2d869b340ef7e020ca0a697834c291b07f3123eb Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Wed, 9 Aug 2023 10:40:36 -0700 Subject: [PATCH 30/41] Rebased from a recent version. --- gpt_engineer/file_selector.py | 3 +-- gpt_engineer/main.py | 19 ++++++------------- gpt_engineer/steps.py | 4 ++-- projects/example/prompt | 2 +- 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py index c93dbdec58..37945bc1ef 100644 --- a/gpt_engineer/file_selector.py +++ b/gpt_engineer/file_selector.py @@ -132,7 +132,7 @@ def __init__(self, root_folder_path: Path) -> None: self.selectable_file_paths: dict[int, str] = {} self.file_path_list: list = [] self.db_paths = DisplayablePath.make_tree( - root_folder_path, parent=None, criteria=self.is_in_ignoring_extensions + root_folder_path, parent=None, criteria=is_in_ignoring_extensions ) def display(self): @@ -208,7 +208,6 @@ def ask_for_selection(self) -> List[str]: return selected_paths - def is_in_ignoring_extensions(path: Path) -> bool: """ Check if a path is not hidden or in the __pycache__ directory. diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index f9585e9c6f..ed823e6499 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -1,13 +1,12 @@ import logging import os -import shutil from pathlib import Path import typer -from gpt_engineer.ai import AI, fallback_model +from gpt_engineer.ai import AI from gpt_engineer.collect import collect_learnings from gpt_engineer.db import DB, DBs, archive from gpt_engineer.learning import collect_consent @@ -37,8 +36,8 @@ def main( input_path = Path(project_path).absolute() - memory_path = input_path / f"{run_prefix}memory" - workspace_path = input_path / f"{run_prefix}workspace" + memory_path = input_path / "memory" + workspace_path = input_path / "workspace" # For the improve option take current project as path and add .gpteng folder # By now, ignoring the 'project_path' argument if improve_option: @@ -47,17 +46,11 @@ def main( # The default option for the --improve is the IMPROVE_CODE, not DEFAULT # I know this looks ugly, not sure if it is the best way to do that... # we can change that in the future. - if steps_config == steps.Config.DEFAULT: - steps_config = steps.Config.IMPROVE_CODE - memory_path = input_path / f"{run_prefix}memory" + if steps_config == StepsConfig.DEFAULT: + steps_config = StepsConfig.IMPROVE_CODE + memory_path = input_path / "memory" workspace_path = Path(os.getcwd()).absolute() - if delete_existing: - # Delete files and subdirectories in paths - shutil.rmtree(memory_path, ignore_errors=True) - shutil.rmtree(workspace_path, ignore_errors=True) - - model = fallback_model(model) ai = AI( model_name=model, temperature=temperature, diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index dace53f121..e55a9c693c 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -310,9 +310,9 @@ def improve_existing_code(ai: AI, dbs: DBs): code_input = format_file_to_input(file_name, file_str) messages.append(ai.fuser(f"{code_input}")) - messages = ai.next(messages) + messages = ai.next(messages, step_name=curr_fn()) # Maybe we should add another step called "replace" or "overwrite" - overwrite_files(messages[-1]["content"], dbs, replace_files=file_path_info) + overwrite_files(messages[-1].content.strip(), dbs, replace_files=file_path_info) return messages diff --git a/projects/example/prompt b/projects/example/prompt index 100c878429..52310b2260 100644 --- a/projects/example/prompt +++ b/projects/example/prompt @@ -1 +1 @@ -We are writing snake in python. MVC components split in separate files. Keyboard control. +The current game boundry is 30x20 in main.py, please make that 40x40. \ No newline at end of file From 3128673ea641072b2de725e8595ea3d88e614f36 Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Wed, 9 Aug 2023 11:47:33 -0700 Subject: [PATCH 31/41] During existing file mode, the file section options start with 1 not 0, and a few common library folders are ignored (site-packages, and node_modules). --- gpt_engineer/file_selector.py | 19 ++++++++++--------- gpt_engineer/main.py | 6 ------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py index 37945bc1ef..df1c5f357e 100644 --- a/gpt_engineer/file_selector.py +++ b/gpt_engineer/file_selector.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import List, Union +IGNORE_FOLDERS = {"site-packages", "node_modules"} class DisplayablePath(object): """ @@ -75,7 +76,7 @@ def make_tree(cls, root: Union[str, Path], parent=None, is_last=False, criteria= count = 1 for path in children: is_last = count == len(children) - if path.is_dir(): + if path.is_dir() and path.name not in IGNORE_FOLDERS: yield from cls.make_tree( path, parent=displayable_root, is_last=is_last, criteria=criteria ) @@ -232,19 +233,19 @@ def ask_for_files(db_input) -> dict[str, str]: dict[str, str]: Dictionary where key = file name and value = file path """ use_last_string = "" - selection_number = 0 + selection_number = 1 is_valid_selection = False can_use_last = False if "file_list.txt" in db_input: can_use_last = True use_last_string = ( - "2. Use previous file list (available at " + "3. Use previous file list (available at " + f"{os.path.join(db_input.path, 'file_list.txt')})\n" ) selection_str = f"""How do you want to select the files? -0. Use Command-Line. -1. Use File explorer. +1. Use Command-Line. +2. Use File explorer. {use_last_string if len(use_last_string) > 1 else ""} Select option and press Enter (default={selection_number}): """ file_path_list = [] @@ -255,16 +256,16 @@ def ask_for_files(db_input) -> dict[str, str]: except ValueError: print("Invalid number. Select a number from the list above.\n") sys.exit(1) - if selection_number == 0: + if selection_number == 1: # Open terminal selection file_path_list = terminal_file_selector() is_valid_selection = True - elif selection_number == 1: + elif selection_number == 2: # Open GUI selection file_path_list = gui_file_selector() is_valid_selection = True else: - if can_use_last and selection_number == 2: + if can_use_last and selection_number == 3: # Use previous file list is_valid_selection = True if not is_valid_selection: @@ -273,7 +274,7 @@ def ask_for_files(db_input) -> dict[str, str]: file_list_string = "" file_path_info = {} - if not selection_number == 2: + if not selection_number == 3: # New files for file_path in file_path_list: file_list_string += str(file_path) + "\n" diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index 4c8c620d5c..ed823e6499 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -29,12 +29,6 @@ def main( "-i", help="Improve code from existing project.", ), - improve_option: bool = typer.Option( - False, - "--improve", - "-i", - help="Improve code from existing project.", - ), verbose: bool = typer.Option(False, "--verbose", "-v"), ): logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO) From 8a9eb332df6962aea9a80d2e007c10aec30ee63e Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Wed, 9 Aug 2023 12:41:13 -0700 Subject: [PATCH 32/41] fixed rebase bugs overwritten by automerge --- gpt_engineer/steps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 23e818a9d8..0404d9f837 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -354,9 +354,9 @@ def improve_existing_code(ai: AI, dbs: DBs): code_input = format_file_to_input(file_name, file_str) messages.append(ai.fuser(f"{code_input}")) - messages = ai.next(messages) + messages = ai.next(messages, step_name=curr_fn()) # Maybe we should add another step called "replace" or "overwrite" - overwrite_files(messages[-1]["content"], dbs, replace_files=file_path_info) + overwrite_files(messages[-1].content.strip(), dbs, replace_files=file_path_info) return messages From 2a529be379fe88227cc862ca09e7fa9d200d8dd1 Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Wed, 9 Aug 2023 14:38:00 -0700 Subject: [PATCH 33/41] In Existing Code mode the default selection for files is file_list.txt if it exists --- gpt_engineer/file_selector.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gpt_engineer/file_selector.py b/gpt_engineer/file_selector.py index df1c5f357e..484a16fad0 100644 --- a/gpt_engineer/file_selector.py +++ b/gpt_engineer/file_selector.py @@ -233,7 +233,6 @@ def ask_for_files(db_input) -> dict[str, str]: dict[str, str]: Dictionary where key = file name and value = file path """ use_last_string = "" - selection_number = 1 is_valid_selection = False can_use_last = False if "file_list.txt" in db_input: @@ -242,6 +241,9 @@ def ask_for_files(db_input) -> dict[str, str]: "3. Use previous file list (available at " + f"{os.path.join(db_input.path, 'file_list.txt')})\n" ) + selection_number = 3 + else: + selection_number = 1 selection_str = f"""How do you want to select the files? 1. Use Command-Line. From f6890945fc0211833d5d25ed6fc4725ae0e26e31 Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Wed, 9 Aug 2023 17:49:16 -0700 Subject: [PATCH 34/41] added additional formatting to improve_existing_code() so that the output format is correct and the regEx parser can pick up the file name and place the file correctly. --- gpt_engineer/steps.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 0404d9f837..eb843222fe 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -354,7 +354,18 @@ def improve_existing_code(ai: AI, dbs: DBs): code_input = format_file_to_input(file_name, file_str) messages.append(ai.fuser(f"{code_input}")) - messages = ai.next(messages, step_name=curr_fn()) + output_format_str = """ +Make sure the output of any files is in the following format where +FILENAME is the file name including the file extension, +LANG is the markup code block language for the code's language, and CODE is the code: + +FILENAME +```LANG +CODE +``` +""" + + messages = ai.next(messages, output_format_str, step_name=curr_fn()) # Maybe we should add another step called "replace" or "overwrite" overwrite_files(messages[-1].content.strip(), dbs, replace_files=file_path_info) return messages From 76c95b553c10e7a4d79d3c0916ef2b6a259c6129 Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Sun, 13 Aug 2023 08:13:52 -0700 Subject: [PATCH 35/41] Update gpt_engineer/main.py Remove development notes. Co-authored-by: Anton Osika --- gpt_engineer/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index ed823e6499..fc96797a3c 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -44,7 +44,6 @@ def main( input_path = Path(os.getcwd()).absolute() / ".gpteng" input_path.mkdir(parents=True, exist_ok=True) # The default option for the --improve is the IMPROVE_CODE, not DEFAULT - # I know this looks ugly, not sure if it is the best way to do that... # we can change that in the future. if steps_config == StepsConfig.DEFAULT: steps_config = StepsConfig.IMPROVE_CODE From 5203a3949f3195a4056eafd9af6a91ce5a4d5035 Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Sun, 13 Aug 2023 08:28:53 -0700 Subject: [PATCH 36/41] Update gpt_engineer/main.py Co-authored-by: Anton Osika --- gpt_engineer/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index fc96797a3c..951e6c488b 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -34,7 +34,6 @@ def main( logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO) - input_path = Path(project_path).absolute() memory_path = input_path / "memory" workspace_path = input_path / "workspace" From 63052f865f2721955b22389454a4459e527d22ec Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Sun, 13 Aug 2023 08:29:06 -0700 Subject: [PATCH 37/41] Update gpt_engineer/main.py Co-authored-by: Anton Osika --- gpt_engineer/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index 951e6c488b..7195b08250 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -35,7 +35,6 @@ def main( - memory_path = input_path / "memory" workspace_path = input_path / "workspace" # For the improve option take current project as path and add .gpteng folder # By now, ignoring the 'project_path' argument From 97931ba19cd2c92dc1c98217583ed1473a4968b7 Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Sun, 13 Aug 2023 08:29:23 -0700 Subject: [PATCH 38/41] Update gpt_engineer/main.py remove redundant code Co-authored-by: Anton Osika --- gpt_engineer/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index 7195b08250..8186407424 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -35,7 +35,6 @@ def main( - workspace_path = input_path / "workspace" # For the improve option take current project as path and add .gpteng folder # By now, ignoring the 'project_path' argument if improve_option: From fbb8d289baa337a299054923723957ce19297ded Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Sun, 13 Aug 2023 08:29:40 -0700 Subject: [PATCH 39/41] Update gpt_engineer/main.py remove development notes Co-authored-by: Anton Osika --- gpt_engineer/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index 8186407424..69a1dbbb22 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -41,7 +41,6 @@ def main( input_path = Path(os.getcwd()).absolute() / ".gpteng" input_path.mkdir(parents=True, exist_ok=True) # The default option for the --improve is the IMPROVE_CODE, not DEFAULT - # we can change that in the future. if steps_config == StepsConfig.DEFAULT: steps_config = StepsConfig.IMPROVE_CODE memory_path = input_path / "memory" From 5e1280ca4cc37ba534da233090230aed6a518d8d Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Sun, 13 Aug 2023 08:30:06 -0700 Subject: [PATCH 40/41] Update projects/example/prompt Co-authored-by: Anton Osika --- projects/example/prompt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/example/prompt b/projects/example/prompt index 52310b2260..7f4d799c3e 100644 --- a/projects/example/prompt +++ b/projects/example/prompt @@ -1 +1 @@ -The current game boundry is 30x20 in main.py, please make that 40x40. \ No newline at end of file +We are writing snake in python. MVC components split in separate files. Keyboard control. \ No newline at end of file From 962d2be541db965c04368e0ccde8ca4aa93c0fab Mon Sep 17 00:00:00 2001 From: Peter Harrington Date: Mon, 14 Aug 2023 08:47:27 -0700 Subject: [PATCH 41/41] Update gpt_engineer/chat_to_files.py Co-authored-by: Anton Osika --- gpt_engineer/chat_to_files.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index c608a9bb24..1b98d09cb7 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -75,7 +75,6 @@ def get_code_strings(input) -> dict[str, str]: files_paths = input["file_list.txt"].strip().split("\n") files_dict = {} for file_path in files_paths: - file_data = "" with open(file_path, "r") as file: file_data = file.read() if file_data: