Make steps configurable

benchmark/currency_converter/main_prompt

@@ -0,0 +1 @@
+Build a currency converter app using an API for exchange rates. Use HTML, CSS, and JavaScript for the frontend and Node.js for the backend. Allow users to convert between different currencies.

benchmark/file_organizer/main_prompt

@@ -0,0 +1 @@
+Create a file organizer CLI tool in Python that sorts files in a directory based on their file types (e.g., images, documents, audio) and moves them into corresponding folders.

benchmark/markdown_editor/main_prompt

@@ -0,0 +1 @@
+Build a simple markdown editor using HTML, CSS, and JavaScript. Allow users to input markdown text and display the formatted output in real-time.

benchmark/password_generator/main_prompt

@@ -0,0 +1 @@
+Create a password generator CLI tool in Python that generates strong, random passwords based on user-specified criteria, such as length and character types (letters, numbers, symbols).

benchmark/pomodoro_timer/main_prompt

@@ -0,0 +1 @@
+Develop a Pomodoro timer app using HTML, CSS, and JavaScript. Allow users to set work and break intervals and receive notifications when it's time to switch.

benchmark/url_shortener/main_prompt

@@ -0,0 +1 @@
+Create a URL shortener app using HTML, CSS, JavaScript, and a backend language like Python or Node.js. Allow users to input a long URL and generate a shortened version that redirects to the original URL. Store the shortened URLs in a database.

benchmark/weather_app/main_prompt

@@ -0,0 +1 @@
+Develop a weather app using Python and a weather API. Display current weather conditions for a given location, including temperature, humidity, and weather description.

gpt_engineer/main.py

@@ -21,6 +21,7 @@ def chat(
     ),
     model: str = "gpt-4",
     temperature: float = 0.1,
+    steps_config: str = "default",
 ):
     app_dir = pathlib.Path(os.path.curdir)
     input_path = project_path
@@ -40,7 +41,7 @@ def chat(
         identity=DB(app_dir / "identity"),
     )
 
-    for step in STEPS:
+    for step in STEPS[steps_config]:
         messages = step(ai, dbs)
         dbs.logs[step.__name__] = json.dumps(messages)
 

gpt_engineer/steps.py

@@ -62,7 +62,11 @@ def run_clarified(ai: AI, dbs: DBs):
     return messages
 
 
-STEPS = [clarify, run_clarified]
+# Different configs of what steps to run
+STEPS = {
+    'default': [run],
+    'clarify': [clarify, run_clarified],
+}
 
 # Future steps that can be added:
 # improve_files,

scripts/benchmark.py

@@ -10,19 +10,26 @@
 import argparse
 import json
 from pathlib import Path
+from typer import run
+from itertools import islice
 
-
-def main():
+def main(
+    n_benchmarks: int | None = None,
+):
     processes = []
     files = []
-    benchmarks = Path('benchmark')
-    for folder in benchmarks.iterdir():
+    path = Path('benchmark')
+
+    if n_benchmarks:
+        benchmarks = islice(path.iterdir(), n_benchmarks)
+
+    for folder in benchmarks:
         if os.path.isdir(folder):
             print('Running benchmark for {}'.format(folder))
 
             log_path = folder / 'log.txt'
             log_file = open(log_path, 'w')
-            processes.append(subprocess.Popen(['python', '-m', 'gpt_engineer.main', folder], stdout=log_file, stderr=log_file))
+            processes.append(subprocess.Popen(['python', '-m', 'gpt_engineer.main', folder], stdout=log_file, stderr=log_file, bufsize=0))
             files.append(log_file)
 
             print('You can stream the log file by running: tail -f {}'.format(log_path))
@@ -34,6 +41,6 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    run(main)