From de6ff17c3f4eb51b2f7c26dfdae3ea02a8f86456 Mon Sep 17 00:00:00 2001
From: carlosejimenez <cjsaltlake@gmail.com>
Date: Sat, 11 Jan 2025 18:57:26 +0000
Subject: [PATCH] Bump to 2.1.8

---
 README.md            | 12 ++++++++++++
 swebench/__init__.py |  2 +-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c4cbe995..336279f7 100644
--- a/README.md
+++ b/README.md
@@ -30,6 +30,7 @@ Code and data for our ICLR 2024 paper <a href="http://swe-bench.github.io/paper.
 Please refer our [website](http://swe-bench.github.io) for the public leaderboard and the [change log](https://github.com/princeton-nlp/SWE-bench/blob/main/CHANGELOG.md) for information on the latest updates to the SWE-bench benchmark.
 
 ## 📰 News
+* **[Jan. 11, 2025]**: Thanks to [Modal](https://modal.com/), we've added a new evaluation mode that runs evaluations entirely on the cloud! See 🚀 Set Up on this page for more details.
 * **[Aug. 13, 2024]**: Introducing *SWE-bench Verified*! Part 2 of our collaboration with [OpenAI Preparedness](https://openai.com/preparedness/). A subset of 500 problems that real software engineers have confirmed are solvable. Check out more in the [report](https://openai.com/index/introducing-swe-bench-verified/)!
 * **[Jun. 27, 2024]**: We have an exciting update for SWE-bench - with support from [OpenAI's Preparedness](https://openai.com/preparedness/) team: We're moving to a fully containerized evaluation harness using Docker for more reproducible evaluations! Read more in our [report](https://github.com/princeton-nlp/SWE-bench/blob/main/docs/20240627_docker/README.md).
 * **[Apr. 15, 2024]**: SWE-bench has gone through major improvements to resolve issues with the evaluation harness. Read more in our [report](https://github.com/princeton-nlp/SWE-bench/blob/main/docs/20240415_eval_bug/README.md).
@@ -69,6 +70,17 @@ python -m swebench.harness.run_evaluation \
     --run_id validate-gold
 ```
 
+### 🌩️ Evaluation with Modal
+You can also run evaluations entirely on the cloud using [Modal](https://modal.com/) to avoid local setup and resource constraints:
+```bash
+python -m swebench.harness.run_evaluation \
+    --predictions_path gold \
+    --run_id validate-gold-modal \
+    --instance_ids sympy__sympy-20590 \
+    --modal true
+```
+This will execute the evaluation harness on Modal's cloud infrastructure, eliminating the need for local Docker setup and resource management.
+
 ## 💽 Usage
 > [!WARNING]
 > Running fast evaluations on SWE-bench can be resource intensive
diff --git a/swebench/__init__.py b/swebench/__init__.py
index 10b51570..96ac08ad 100644
--- a/swebench/__init__.py
+++ b/swebench/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "2.1.7"
+__version__ = "2.1.8"
 
 from swebench.collect.build_dataset import main as build_dataset
 from swebench.collect.get_tasks_pipeline import main as get_tasks_pipeline