inc concurrency for writing to the db

getsavvyinc · May 12, 2024 · 2bfcf75 · 2bfcf75
1 parent 4805e01
commit 2bfcf75
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/ingest/main.go b/ingest/main.go
@@ -23,7 +23,7 @@ import (
 const tldrPath = "tldr/pages/"
 
 const maxConcurrency = 500
-const maxDBConcurrency = 50
+const maxDBConcurrency = 75
 
 func main() {
 	logger := slog.Default()
@@ -100,7 +100,7 @@ func main() {
 		if items-lastIndex >= 100 && items != 0 {
 			// wait a bit to avoid hitting the open ai rate limit
 			logger.Info("sleeping for two seconds", "cheatsheet", items)
-			time.Sleep(10 * time.Second)
+			time.Sleep(5 * time.Second)
 			lastIndex = items
 		}