Merge pull request #8 from DrBlury/main

Added go concurrency alternative

.gitignore

@@ -1,5 +1,7 @@
 rust/target
 rust_rayon/target
 go/related
+go_con/related_concurrent
 odin/main
 related*.json
+.DS_Store

go.work

@@ -0,0 +1,6 @@
+go 1.21.1
+
+use (
+	./go
+	./go_con
+)

go_con/go.mod

@@ -0,0 +1,7 @@
+module g.io/related_concurrent
+
+go 1.21.1
+
+require github.com/goccy/go-json v0.10.2
+
+require github.com/ugurcsen/gods-generic v0.10.4 // indirect

go_con/go.sum

@@ -0,0 +1,4 @@
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/ugurcsen/gods-generic v0.10.4 h1:OomH3R2MdzZxpnEPijaD/ncLzV6rpDXd5ruEkWsw0vo=
+github.com/ugurcsen/gods-generic v0.10.4/go.mod h1:mGYOa88Y5sbw+ADXLpScxjJ7s5iHoWya/YHyeQ4f6c4=

go_con/main.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/goccy/go-json"
+	"github.com/ugurcsen/gods-generic/trees/binaryheap"
+)
+
+var concurrency = runtime.NumCPU()
+
+type Post struct {
+	ID    string   `json:"_id"`
+	Title string   `json:"title"`
+	Tags  []string `json:"tags"`
+}
+
+type PostWithSharedTags struct {
+	Post       int
+	SharedTags int
+}
+
+type RelatedPosts struct {
+	ID      string   `json:"_id"`
+	Tags    []string `json:"tags"`
+	Related []*Post  `json:"related"`
+}
+
+// Result struct to hold results from goroutines
+type Result struct {
+	Index       int
+	RelatedPost RelatedPosts
+}
+
+func main() {
+	f, err := os.Open("../posts.json")
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	var posts []Post
+	err = json.NewDecoder(f).Decode(&posts)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	start := time.Now()
+
+	tagMap := make(map[string][]int, 100)
+	for i, post := range posts {
+		for _, tag := range post.Tags {
+			tagMap[tag] = append(tagMap[tag], i)
+		}
+	}
+
+	resultsChan := make(chan Result, len(posts))
+	doneChan := make(chan bool, concurrency)
+
+	for w := 0; w < concurrency; w++ {
+		go func(workerID int) {
+			defer func() { doneChan <- true }()
+			for i := workerID; i < len(posts); i += concurrency {
+				relatedPost := computeRelatedPost(i, posts, tagMap)
+				resultsChan <- Result{Index: i, RelatedPost: relatedPost}
+			}
+		}(w)
+	}
+
+	for i := 0; i < concurrency; i++ {
+		<-doneChan
+	}
+	close(resultsChan)
+	close(doneChan)
+
+	allRelatedPosts := make([]RelatedPosts, len(posts))
+	for r := range resultsChan {
+		allRelatedPosts[r.Index] = r.RelatedPost
+	}
+
+	end := time.Now()
+
+	fmt.Println("Processing time (w/o IO)", end.Sub(start))
+
+	file, err := os.Create("../related_posts_go_con.json")
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	err = json.NewEncoder(file).Encode(allRelatedPosts)
+	if err != nil {
+		log.Panicln(err)
+	}
+}
+
+func computeRelatedPost(i int, posts []Post, tagMap map[string][]int) RelatedPosts {
+	taggedPostCount := make([]int, len(posts))
+	t5 := binaryheap.NewWith[PostWithSharedTags](PostComparator)
+
+	for _, tag := range posts[i].Tags {
+		for _, otherPostIdx := range tagMap[tag] {
+			if otherPostIdx != i {
+				taggedPostCount[otherPostIdx]++
+			}
+		}
+	}
+
+	for v, count := range taggedPostCount {
+		if t5.Size() < 5 {
+			t5.Push(PostWithSharedTags{Post: v, SharedTags: count})
+		} else {
+			if t, _ := t5.Peek(); t.SharedTags < count {
+				t5.Pop()
+				t5.Push(PostWithSharedTags{Post: v, SharedTags: count})
+			}
+		}
+	}
+
+	num := min(5, t5.Size())
+	topPosts := make([]*Post, num)
+
+	for i := 0; i < num; i++ {
+		if t, ok := t5.Pop(); ok {
+			topPosts[i] = &posts[t.Post]
+		}
+	}
+
+	return RelatedPosts{
+		ID:      posts[i].ID,
+		Tags:    posts[i].Tags,
+		Related: topPosts,
+	}
+}
+
+func PostComparator(a, b PostWithSharedTags) int {
+	if a.SharedTags > b.SharedTags {
+		return 1
+	}
+	if a.SharedTags < b.SharedTags {
+		return -1
+	}
+	return 0
+}

run.sh

@@ -13,20 +13,31 @@ run_go() {
         cd ./go &&
         go build &&
         if [ $HYPER == 1 ]; then
-            command hyperfine -r 10 --show-output "./related"
+            command hyperfine --show-output -r 10 -w 3 "./related"
         else
             command time -f '%es %Mk' ./related
         fi
 }
 
+run_go_concurrent() {
+    echo "Running Go with concurrency" &&
+        cd ./go_con &&
+        go build &&
+        if [ $HYPER == 1 ]; then
+            command hyperfine --show-output -r 10 -w 3 "./related_concurrent"
+        else
+            command time -f '%es %Mk' ./related_concurrent
+        fi
+}
+
 run_rust() {
     echo "Running Rust" &&
         cd ./rust &&
         cargo build --release &&
         if [ $HYPER == 1 ]; then
-            command hyperfine -r 20 --show-output "./target/release/rust"
+            command hyperfine --show-output -r 10 -w 3 "./target/release/rust"
         else
-            command time -f '%es %Mk' ./target/release/rust
+            command ./target/release/rust
         fi
 }
 
@@ -35,7 +46,7 @@ run_rust_rayon() {
         cd ./rust_rayon &&
         cargo build --release &&
         if [ $HYPER == 1 ]; then
-            command hyperfine -r 10 --show-output "./target/release/rust_rayon"
+            command hyperfine --show-output -r 10 -w 3 "./target/release/rust_rayon"
         else
             command time -f '%es %Mk' ./target/release/rust_rayon
         fi
@@ -45,7 +56,7 @@ run_python() {
     echo "Running Python" &&
         cd ./python &&
         if [ $HYPER == 1 ]; then
-            command hyperfine -r 2 --show-output "python3 ./related.py"
+            command hyperfine -r 1 "python3 ./related.py"
         else
             command time -f '%es %Mk' python3 ./related.py
         fi
@@ -62,6 +73,11 @@ if [ "$first_arg" = "go" ]; then
     run_go &&
         check_output "related_posts_go.json"
 
+elif [ "$first_arg" = "go_con" ]; then
+
+    run_go_concurrent &&
+        check_output "related_posts_go_con.json"
+
 elif [ "$first_arg" = "rust" ]; then
 
     run_rust &&
@@ -84,6 +100,8 @@ elif [ "$first_arg" = "all" ]; then
         cd .. &&
         run_rust &&
         cd .. &&
+        run_rust_rayon &&
+        cd .. &&
         run_python
 
 else