Merge pull request #5 from anon998/stop-stream

Stop generating tokens when the stream is closed.
SlyEcho · May 31, 2023 · 9f2424a · 9f2424a
2 parents b6f536d + 3a079d5
commit 9f2424a
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
@@ -668,7 +668,7 @@ bool parse_options_completion(json body, llama_server_context& llama, Response &
       std::string tmp_stop =
           std::accumulate(llama.params.antiprompt.begin(), llama.params.antiprompt.end(),
                           std::string{}, [](std::string a, std::string b) {
-                              return a + (a != "" ? ", \"" : "") + b + "\"";
+                              return a + (a != "" ? ", \"" : "\"") + b + "\"";
                           });
 
       fprintf(stderr,
@@ -860,7 +860,12 @@ int main(int argc, char **argv)
                                     data.dump(llama.json_indent, ' ', false,
                                               json::error_handler_t::replace) +
                                     "\n\n";
-                  sink.write(str.data(), str.size());
+                  if (!sink.write(str.data(), str.size())) {
+                      if (llama.verbose) {
+                          fprintf(stderr, "stream closed\n");
+                      }
+                      return false;
+                  }
               }
 
               sink.done();