Update tokenizer.go

sea-team · Aug 17, 2022 · 477af1c · 477af1c
1 parent 12769cf
commit 477af1c
Showing 1 changed file with 4 additions and 7 deletions.
diff --git a/searcher/words/tokenizer.go b/searcher/words/tokenizer.go
@@ -42,9 +42,10 @@ func (t *Tokenizer) Cut(text string) []string {
 	//移除所有的空格
 	text = utils.RemoveSpace(text)
 
-	var wordMap = make(map[string]int)
+	var wordMap = make(map[string]struct{})
 
 	resultChan := t.seg.CutForSearch(text, true)
+	var wordsSlice []string
 	for {
 		w, ok := <-resultChan
 		if !ok {
@@ -53,14 +54,10 @@ func (t *Tokenizer) Cut(text string) []string {
 		_, found := wordMap[w]
 		if !found {
 			//去除重复的词
-			wordMap[w] = 1
+			wordMap[w] = struct{}{}
+			wordsSlice = append(wordsSlice, w)
 		}
 	}
 
-	var wordsSlice []string
-	for k := range wordMap {
-		wordsSlice = append(wordsSlice, k)
-	}
-
 	return wordsSlice
 }