diff --git a/.circleci/config.yml b/.circleci/config.yml index 65e760b..f6b75d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,7 @@ jobs: - run: name: Generate a root CA and a server certificate using redis helpers command: | - git clone git://github.com/antirez/redis.git --branch 6.0.5 + git clone git://github.com/antirez/redis.git --branch 6.0.6 cd redis ./utils/gen-test-certs.sh cd .. @@ -53,9 +53,41 @@ jobs: - checkout - run: make get - run: make checkfmt + - run: make test + - run: make godoc_examples - run: make coverage - run: bash <(curl -s https://codecov.io/bash) -t ${CODECOV_TOKEN} + build-latest: + docker: + - image: circleci/golang:1.12 + - image: redislabs/redisearch:latest + + working_directory: /go/src/github.com/RediSearch/redisearch-go + steps: + - checkout + - run: make test + + build-v16: + docker: + - image: circleci/golang:1.12 + - image: redislabs/redisearch:1.6.13 + + working_directory: /go/src/github.com/RediSearch/redisearch-go + steps: + - checkout + - run: make test + + build-v14: + docker: + - image: circleci/golang:1.12 + - image: redislabs/redisearch:1.4.28 + + working_directory: /go/src/github.com/RediSearch/redisearch-go + steps: + - checkout + - run: make test + build_nightly: # test nightly with redisearch:edge docker: - image: circleci/golang:1.12 @@ -64,7 +96,6 @@ jobs: working_directory: /go/src/github.com/RediSearch/redisearch-go steps: - checkout - - run: make get - run: make test workflows: @@ -73,6 +104,9 @@ workflows: jobs: - build - build-tls + - build-latest + - build-v16 + - build-v14 nightly: triggers: - schedule: @@ -83,4 +117,7 @@ workflows: - master jobs: - build_nightly - - build-tls \ No newline at end of file + - build-tls + - build-latest + - build-v16 + - build-v14 \ No newline at end of file diff --git a/Makefile b/Makefile index 629d544..8a732df 100644 --- a/Makefile +++ b/Makefile @@ -41,10 +41,15 @@ examples: get --tls-ca-cert-file $(TLS_CACERT) \ --host $(REDISEARCH_TEST_HOST) -test: get +fmt: $(GOFMT) ./... - $(GOTEST) -race -covermode=atomic ./... -coverage: get test - $(GOTEST) -race -coverprofile=coverage.txt -covermode=atomic ./redisearch +godoc_examples: get fmt + $(GOTEST) -race -covermode=atomic -v ./redisearch + +test: get fmt + $(GOTEST) -race -covermode=atomic -run "Test" -v ./redisearch + +coverage: get + $(GOTEST) -race -coverprofile=coverage.txt -covermode=atomic -v ./redisearch diff --git a/README.md b/README.md index 335c58c..565e890 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,6 @@ func ExampleClient() { | :--- | ----: | | [FT.CREATE](https://oss.redislabs.com/redisearch/Commands.html#ftcreate) | [CreateIndex](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.CreateIndex) | | [FT.ADD](https://oss.redislabs.com/redisearch/Commands.html#ftadd) | [IndexOptions](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.IndexOptions) | -| [FT.ADDHASH](https://oss.redislabs.com/redisearch/Commands.html#ftaddhash) | [AddHash](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.AddHash) | | [FT.ALTER](https://oss.redislabs.com/redisearch/Commands.html#ftalter) | [AddField](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.AddField) | | [FT.ALIASADD](https://oss.redislabs.com/redisearch/Commands.html#ftaliasadd) | [AliasAdd](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.AliasAdd) | | [FT.ALIASUPDATE](https://oss.redislabs.com/redisearch/Commands.html#ftaliasupdate) | [AliasUpdate](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.AliasUpdate) | @@ -97,7 +96,6 @@ func ExampleClient() { | [FT.SUGGET](https://oss.redislabs.com/redisearch/Commands.html#ftsugget) | [SuggestOpts](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Autocompleter.SuggestOpts) | | [FT.SUGDEL](https://oss.redislabs.com/redisearch/Commands.html#ftsugdel) | [DeleteTerms](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Autocompleter.DeleteTerms) | | [FT.SUGLEN](https://oss.redislabs.com/redisearch/Commands.html#ftsuglen) | [Autocompleter.Length](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Autocompleter.Length) | -| [FT.SYNADD](https://oss.redislabs.com/redisearch/Commands.html#ftsynadd) | [SynAdd](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.SynAdd) | | [FT.SYNUPDATE](https://oss.redislabs.com/redisearch/Commands.html#ftsynupdate) | [SynUpdate](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.SynUpdate) | | [FT.SYNDUMP](https://oss.redislabs.com/redisearch/Commands.html#ftsyndump) | [SynDump](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.SynDump) | | [FT.SPELLCHECK](https://oss.redislabs.com/redisearch/Commands.html#ftspellcheck) | [SpellCheck](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.SpellCheck) | diff --git a/redisearch/client.go b/redisearch/client.go index 8cce654..bdffa4b 100644 --- a/redisearch/client.go +++ b/redisearch/client.go @@ -46,19 +46,32 @@ func NewClientFromPool(pool *redis.Pool, name string) *Client { return ret } -// CreateIndex configues the index and creates it on redis -func (i *Client) CreateIndex(s *Schema) (err error) { +// CreateIndex configures the index and creates it on redis +func (i *Client) CreateIndex(schema *Schema) (err error) { + return i.indexWithDefinition(i.name, schema, nil) +} + +// CreateIndexWithIndexDefinition configures the index and creates it on redis +// IndexDefinition is used to define a index definition for automatic indexing on Hash update +func (i *Client) CreateIndexWithIndexDefinition(schema *Schema, definition *IndexDefinition) (err error) { + return i.indexWithDefinition(i.name, schema, definition) +} + +// internal method +func (i *Client) indexWithDefinition(indexName string, schema *Schema, definition *IndexDefinition) (err error) { args := redis.Args{i.name} + if definition != nil { + args = definition.Serialize(args) + } // Set flags based on options - args, err = SerializeSchema(s, args) + args, err = SerializeSchema(schema, args) if err != nil { return } - conn := i.pool.Get() defer conn.Close() _, err = conn.Do("FT.CREATE", args...) - return err + return } // AddField Adds a new field to the index. @@ -340,7 +353,7 @@ func (i *Client) Explain(q *Query) (string, error) { return redis.String(conn.Do("FT.EXPLAIN", args...)) } -// Drop the Currentl just flushes the DB - note that this will delete EVERYTHING on the redis instance +// Deletes the index and all the keys associated with it. func (i *Client) Drop() error { conn := i.pool.Get() defer conn.Close() @@ -380,6 +393,13 @@ func (info *IndexInfo) setTarget(key string, value interface{}) error { case reflect.Float64: f, _ := redis.Float64(value, nil) targetInfo.SetFloat(f) + case reflect.Bool: + f, _ := redis.Uint64(value, nil) + if f == 0 { + targetInfo.SetBool(false) + } else { + targetInfo.SetBool(true) + } default: panic("Tag set without handler") } @@ -554,6 +574,7 @@ func (i *Client) GetTagVals(index string, filedName string) ([]string, error) { } // Adds a synonym group. +// Deprecated: This function is not longer supported on RediSearch 2.0 and above, use SynUpdate instead func (i *Client) SynAdd(indexName string, terms []string) (int64, error) { conn := i.pool.Get() defer conn.Close() @@ -562,7 +583,7 @@ func (i *Client) SynAdd(indexName string, terms []string) (int64, error) { return redis.Int64(conn.Do("FT.SYNADD", args...)) } -// Updates a synonym group. +// Updates a synonym group, with additional terms. func (i *Client) SynUpdate(indexName string, synonymGroupId int64, terms []string) (string, error) { conn := i.pool.Get() defer conn.Close() @@ -600,6 +621,8 @@ func (i *Client) SynDump(indexName string) (map[string][]int64, error) { } // Adds a document to the index from an existing HASH key in Redis. +// Deprecated: This function is not longer supported on RediSearch 2.0 and above, use HSET instead +// See the example ExampleClient_CreateIndexWithIndexDefinition for a deeper understanding on how to move towards using hashes on your application func (i *Client) AddHash(docId string, score float32, language string, replace bool) (string, error) { conn := i.pool.Get() defer conn.Close() diff --git a/redisearch/client_test.go b/redisearch/client_test.go index a527ca0..d928279 100644 --- a/redisearch/client_test.go +++ b/redisearch/client_test.go @@ -15,6 +15,38 @@ func flush(c *Client) (err error) { return conn.Send("FLUSHALL") } +func teardown(c *Client) { + flush(c) +} + +// getRediSearchVersion returns RediSearch version by issuing "MODULE LIST" command +// and iterating through the availabe modules up until "ft" is found as the name property +func (c *Client) getRediSearchVersion() (version int64, err error) { + conn := c.pool.Get() + defer conn.Close() + var values []interface{} + var moduleInfo []interface{} + var moduleName string + values, err = redis.Values(conn.Do("MODULE", "LIST")) + if err != nil { + return + } + for _, rawModule := range values { + moduleInfo, err = redis.Values(rawModule, err) + if err != nil { + return + } + moduleName, err = redis.String(moduleInfo[1], err) + if err != nil { + return + } + if moduleName == "ft" { + version, err = redis.Int64(moduleInfo[3], err) + } + } + return +} + func TestClient_Get(t *testing.T) { c := createClient("test-get") @@ -75,6 +107,7 @@ func TestClient_Get(t *testing.T) { }) } + teardown(c) } func TestClient_MultiGet(t *testing.T) { @@ -136,6 +169,7 @@ func TestClient_MultiGet(t *testing.T) { } }) } + teardown(c) } func TestClient_DictAdd(t *testing.T) { @@ -180,6 +214,7 @@ func TestClient_DictAdd(t *testing.T) { i.DictDel(tt.args.dictionaryName, tt.args.terms) }) } + teardown(c) } func TestClient_DictDel(t *testing.T) { @@ -230,6 +265,7 @@ func TestClient_DictDel(t *testing.T) { } }) } + teardown(c) } func TestClient_DictDump(t *testing.T) { @@ -276,6 +312,7 @@ func TestClient_DictDump(t *testing.T) { } }) } + teardown(c) } func TestClient_AliasAdd(t *testing.T) { @@ -323,6 +360,7 @@ func TestClient_AliasAdd(t *testing.T) { } }) } + teardown(c) } func TestClient_AliasDel(t *testing.T) { @@ -373,6 +411,7 @@ func TestClient_AliasDel(t *testing.T) { } }) } + teardown(c) } func TestClient_AliasUpdate(t *testing.T) { @@ -420,6 +459,7 @@ func TestClient_AliasUpdate(t *testing.T) { } }) } + teardown(c) } func TestClient_Config(t *testing.T) { @@ -435,6 +475,7 @@ func TestClient_Config(t *testing.T) { kvs, _ = c.GetConfig("*") assert.Equal(t, "100", kvs["TIMEOUT"]) + teardown(c) } func TestNewClientFromPool(t *testing.T) { @@ -449,6 +490,7 @@ func TestNewClientFromPool(t *testing.T) { err2 := client2.pool.Close() assert.Nil(t, err1) assert.Nil(t, err2) + teardown(client1) } func TestClient_GetTagVals(t *testing.T) { @@ -475,48 +517,64 @@ func TestClient_GetTagVals(t *testing.T) { tags, err = c.GetTagVals("notexit", "tags") assert.NotNil(t, err) assert.Nil(t, tags) + teardown(c) } func TestClient_SynAdd(t *testing.T) { c := createClient("testsynadd") - - sc := NewSchema(DefaultOptions). - AddField(NewTextField("name")). - AddField(NewTextField("addr")) - c.Drop() - err := c.CreateIndex(sc) + version, err := c.getRediSearchVersion() assert.Nil(t, err) - - gid, err := c.SynAdd("testsynadd", []string{"girl", "baby"}) - assert.Nil(t, err) - assert.True(t, gid >= 0) - ret, err := c.SynUpdate("testsynadd", gid, []string{"girl", "baby"}) - assert.Nil(t, err) - assert.Equal(t, "OK", ret) + if version <= 10699 { + sc := NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")) + c.Drop() + err := c.CreateIndex(sc) + assert.Nil(t, err) + + gid, err := c.SynAdd("testsynadd", []string{"girl", "baby"}) + assert.Nil(t, err) + assert.True(t, gid >= 0) + ret, err := c.SynUpdate("testsynadd", gid, []string{"girl", "baby"}) + assert.Nil(t, err) + assert.Equal(t, "OK", ret) + } + teardown(c) } func TestClient_SynDump(t *testing.T) { c := createClient("testsyndump") - + version, err := c.getRediSearchVersion() + assert.Nil(t, err) sc := NewSchema(DefaultOptions). AddField(NewTextField("name")). AddField(NewTextField("addr")) c.Drop() - err := c.CreateIndex(sc) - assert.Nil(t, err) + err = c.CreateIndex(sc) + var gId1 int64 = 1 + var gId2 int64 = 2 - gid, err := c.SynAdd("testsyndump", []string{"girl", "baby"}) assert.Nil(t, err) - assert.True(t, gid >= 0) - - gid2, err := c.SynAdd("testsyndump", []string{"child"}) + // For RediSearch < v2.0 we need to use SYNADD. For Redisearch >= v2.0 we need to use SYNUPDATE + if version <= 10699 { + gId1, err = c.SynAdd("testsyndump", []string{"girl", "baby"}) + assert.Nil(t, err) + gId2, err = c.SynAdd("testsyndump", []string{"child"}) + assert.Nil(t, err) + } else { + ret, err := c.SynUpdate("testsyndump", gId1, []string{"girl", "baby"}) + assert.Nil(t, err) + assert.Equal(t, "OK", ret) + _, err = c.SynUpdate("testsyndump", gId2, []string{"child"}) + assert.Nil(t, err) + assert.Equal(t, "OK", ret) + } m, err := c.SynDump("testsyndump") assert.Contains(t, m, "baby") assert.Contains(t, m, "girl") assert.Contains(t, m, "child") - assert.Equal(t, gid, m["baby"][0]) - assert.Equal(t, gid2, m["child"][0]) + teardown(c) } func TestClient_AddHash(t *testing.T) { @@ -539,6 +597,7 @@ func TestClient_AddHash(t *testing.T) { } else { assert.Equal(t, "OK", ret) } + teardown(c) } func TestClient_AddField(t *testing.T) { @@ -553,33 +612,115 @@ func TestClient_AddField(t *testing.T) { assert.Nil(t, err) err = c.Index(NewDocument("doc-n1", 1.0).Set("age", 15)) assert.Nil(t, err) + teardown(c) } -func TestClient_CreateIndex(t *testing.T) { - type fields struct { - pool ConnPool - name string +func TestClient_GetRediSearchVersion(t *testing.T) { + c := createClient("version-test") + _, err := c.getRediSearchVersion() + assert.Nil(t, err) +} + +func TestClient_CreateIndexWithIndexDefinition(t *testing.T) { + i := createClient("index-definition-test") + version, err := i.getRediSearchVersion() + assert.Nil(t, err) + if version >= 20000 { + + type args struct { + schema *Schema + definition *IndexDefinition + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"no-indexDefinition", args{NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")), nil}, false}, + {"default-indexDefinition", args{NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")), NewIndexDefinition()}, false}, + {"score-indexDefinition", args{NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")), NewIndexDefinition().SetScore(0.25)}, false}, + {"language-indexDefinition", args{NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")), NewIndexDefinition().SetLanguage("portuguese")}, false}, + {"language_field-indexDefinition", args{NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("lang")). + AddField(NewTextField("addr")), NewIndexDefinition().SetLanguageField("lang")}, false}, + {"score_field-indexDefinition", args{NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")).AddField(NewNumericField("score")), NewIndexDefinition().SetScoreField("score")}, false}, + {"payload_field-indexDefinition", args{NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")).AddField(NewNumericField("score")).AddField(NewTextField("payload")), NewIndexDefinition().SetPayloadField("payload")}, false}, + {"prefix-indexDefinition", args{NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")).AddField(NewNumericField("score")).AddField(NewTextField("payload")), NewIndexDefinition().AddPrefix("doc:*")}, false}, + {"filter-indexDefinition", args{NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")).AddField(NewNumericField("score")).AddField(NewTextField("payload")), NewIndexDefinition().SetFilterExpression("@score > 0")}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := i.CreateIndexWithIndexDefinition(tt.args.schema, tt.args.definition); (err != nil) != tt.wantErr { + t.Errorf("CreateIndexWithIndexDefinition() error = %v, wantErr %v", err, tt.wantErr) + } + teardown(i) + }) + } } +} + +func TestClient_SynUpdate(t *testing.T) { + c := createClient("syn-update-test") + sc := NewSchema(DefaultOptions). + AddField(NewTextField("name")). + AddField(NewTextField("addr")) + version, err := c.getRediSearchVersion() + assert.Nil(t, err) + type args struct { - s *Schema + indexName string + synonymGroupId int64 + terms []string } tests := []struct { name string - fields fields args args + want string wantErr bool }{ - // TODO: Add test cases. + {"1-syn", args{"syn-update-test", 1, []string{"abc"}}, "OK", false}, + {"3-syn", args{"syn-update-test", 1, []string{"abc", "def", "ghi"}}, "OK", false}, + {"err-empty-syn", args{"syn-update-test", 1, []string{}}, "", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - i := &Client{ - pool: tt.fields.pool, - name: tt.fields.name, + c.Drop() + err := c.CreateIndex(sc) + assert.Nil(t, err) + gId := tt.args.synonymGroupId + + // For older version of RediSearch we first need to use SYNADD then SYNUPDATE + if version <= 10699 { + gId, err = c.SynAdd(tt.args.indexName, []string{"workaround"}) + assert.Nil(t, err) + } + + got, err := c.SynUpdate(tt.args.indexName, gId, tt.args.terms) + if (err != nil) != tt.wantErr { + t.Errorf("SynUpdate() error = %v, wantErr %v", err, tt.wantErr) + return } - if err := i.CreateIndex(tt.args.s); (err != nil) != tt.wantErr { - t.Errorf("CreateIndex() error = %v, wantErr %v", err, tt.wantErr) + if got != tt.want { + t.Errorf("SynUpdate() got = %v, want %v", got, tt.want) } + teardown(c) }) } } diff --git a/redisearch/document.go b/redisearch/document.go index e39b9bb..1801571 100644 --- a/redisearch/document.go +++ b/redisearch/document.go @@ -27,6 +27,7 @@ type IndexingOptions struct { Language string // If set to true, we will not save the actual document in the database and only index it. + // As of RediSearch 2.0 and above NOSAVE is no longer supported, and will have no effect NoSave bool // If set, we will do an UPSERT style insertion - and delete an older version of the document if it exists. diff --git a/redisearch/example_client_test.go b/redisearch/example_client_test.go index cc962a4..0914437 100644 --- a/redisearch/example_client_test.go +++ b/redisearch/example_client_test.go @@ -44,6 +44,13 @@ func ExampleNewClient() { log.Fatal(err) } + // Wait for all documents to be indexed + info, _ := c.Info() + for info.IsIndexing { + time.Sleep(time.Second) + info, _ = c.Info() + } + // Searching with limit and sorting docs, total, err := c.Search(redisearch.NewQuery("hello world"). Limit(0, 2). @@ -51,6 +58,53 @@ func ExampleNewClient() { fmt.Println(docs[0].Id, docs[0].Properties["title"], total, err) // Output: ExampleNewClient:doc1 Hello world 1 + + // Drop the existing index + c.Drop() +} + +// RediSearch 2.0, marks the re-architecture of the way indices are kept in sync with the data. +// Instead of having to write data through the index (using the FT.ADD command), +// RediSearch will now follow the data written in hashes and automatically index it. +// The following example illustrates how to achieve it with the go client +func ExampleClient_CreateIndexWithIndexDefinition() { + host := "localhost:6379" + password := "" + pool := &redis.Pool{Dial: func() (redis.Conn, error) { + return redis.Dial("tcp", host, redis.DialPassword(password)) + }} + c := redisearch.NewClientFromPool(pool, "products-from-hashes") + + // Create a schema + schema := redisearch.NewSchema(redisearch.DefaultOptions). + AddField(redisearch.NewTextFieldOptions("name", redisearch.TextFieldOptions{Sortable: true})). + AddField(redisearch.NewTextFieldOptions("description", redisearch.TextFieldOptions{Weight: 5.0, Sortable: true})). + AddField(redisearch.NewNumericField("price")) + + // IndexDefinition is available for RediSearch 2.0+ + // Create a index definition for automatic indexing on Hash updates. + // In this example we will only index keys started by product: + indexDefinition := redisearch.NewIndexDefinition().AddPrefix("product:") + + // Add the Index Definition + c.CreateIndexWithIndexDefinition(schema, indexDefinition) + + // Get a vanilla connection and create 100 hashes + vanillaConnection := pool.Get() + for productNumber := 0; productNumber < 100; productNumber++ { + vanillaConnection.Do("HSET", fmt.Sprintf("product:%d", productNumber), "name", fmt.Sprintf("product name %d", productNumber), "description", "product description", "price", 10.99) + } + + // Wait for all documents to be indexed + info, _ := c.Info() + for info.IsIndexing { + time.Sleep(time.Second) + info, _ = c.Info() + } + + _, total, _ := c.Search(redisearch.NewQuery("description")) + + fmt.Printf("Total documents containing \"description\": %d.\n", total) } // exemplifies the NewClientFromPool function @@ -94,6 +148,9 @@ func ExampleNewClientFromPool() { fmt.Println(docs[0].Id, docs[0].Properties["title"], total, err) // Output: ExampleNewClientFromPool:doc2 Hello world 1 + + // Drop the existing index + c.Drop() } //Example of how to establish an SSL connection from your app to the RedisAI Server @@ -180,6 +237,9 @@ func ExampleNewClientFromPool_ssl() { SetReturnFields("title")) fmt.Println(docs[0].Id, docs[0].Properties["title"], total, err) + + // Drop the existing index + c.Drop() } func getConnectionDetails() (host string, password string) { diff --git a/redisearch/filter.go b/redisearch/filter.go index 16e0caf..2e19463 100644 --- a/redisearch/filter.go +++ b/redisearch/filter.go @@ -6,7 +6,7 @@ type Filter struct { Options interface{} } -// Filter the results to a given radius from lon and lat. Radius is given as a number and units +// FilterExpression the results to a given radius from lon and lat. Radius is given as a number and units type GeoFilterOptions struct { Lon float64 Lat float64 diff --git a/redisearch/index.go b/redisearch/index.go new file mode 100644 index 0000000..3fffae2 --- /dev/null +++ b/redisearch/index.go @@ -0,0 +1,163 @@ +package redisearch + +import ( + "github.com/gomodule/redigo/redis" +) + +// IndexInfo - Structure showing information about an existing index +type IndexInfo struct { + Schema Schema + Name string `redis:"index_name"` + DocCount uint64 `redis:"num_docs"` + RecordCount uint64 `redis:"num_records"` + TermCount uint64 `redis:"num_terms"` + MaxDocID uint64 `redis:"max_doc_id"` + InvertedIndexSizeMB float64 `redis:"inverted_sz_mb"` + OffsetVectorSizeMB float64 `redis:"offset_vector_sz_mb"` + DocTableSizeMB float64 `redis:"doc_table_size_mb"` + KeyTableSizeMB float64 `redis:"key_table_size_mb"` + RecordsPerDocAvg float64 `redis:"records_per_doc_avg"` + BytesPerRecordAvg float64 `redis:"bytes_per_record_avg"` + OffsetsPerTermAvg float64 `redis:"offsets_per_term_avg"` + OffsetBitsPerTermAvg float64 `redis:"offset_bits_per_record_avg"` + IsIndexing bool `redis:"indexing"` + PercentIndexed float64 `redis:"percent_indexed"` +} + +// IndexDefinition is used to define a index definition for automatic indexing on Hash update +// This is only valid for >= RediSearch 2.0 +type IndexDefinition struct { + IndexOn string + Async bool + Prefix []string + FilterExpression string + Language string + LanguageField string + Score float64 + ScoreField string + PayloadField string +} + +// This is only valid for >= RediSearch 2.0 +func NewIndexDefinition() *IndexDefinition { + prefixArray := make([]string, 0) + return &IndexDefinition{"HASH", false, prefixArray, "", "", "", -1, "", ""} +} + +// This is only valid for >= RediSearch 2.0 +func (defintion *IndexDefinition) SetAsync(value bool) (outDef *IndexDefinition) { + outDef = defintion + outDef.Async = value + return +} + +// This is only valid for >= RediSearch 2.0 +func (defintion *IndexDefinition) AddPrefix(prefix string) (outDef *IndexDefinition) { + outDef = defintion + outDef.Prefix = append(outDef.Prefix, prefix) + return +} + +func (defintion *IndexDefinition) SetFilterExpression(value string) (outDef *IndexDefinition) { + outDef = defintion + outDef.FilterExpression = value + return +} + +// This is only valid for >= RediSearch 2.0 +func (defintion *IndexDefinition) SetLanguage(value string) (outDef *IndexDefinition) { + outDef = defintion + outDef.Language = value + return +} + +// This is only valid for >= RediSearch 2.0 +func (defintion *IndexDefinition) SetLanguageField(value string) (outDef *IndexDefinition) { + outDef = defintion + outDef.LanguageField = value + return +} + +// This is only valid for >= RediSearch 2.0 +func (defintion *IndexDefinition) SetScore(value float64) (outDef *IndexDefinition) { + outDef = defintion + outDef.Score = value + return +} + +// This is only valid for >= RediSearch 2.0 +func (defintion *IndexDefinition) SetScoreField(value string) (outDef *IndexDefinition) { + outDef = defintion + outDef.ScoreField = value + return +} + +// This is only valid for >= RediSearch 2.0 +func (defintion *IndexDefinition) SetPayloadField(value string) (outDef *IndexDefinition) { + outDef = defintion + outDef.PayloadField = value + return +} + +// This is only valid for >= RediSearch 2.0 +func (defintion *IndexDefinition) Serialize(args redis.Args) redis.Args { + args = append(args, "ON", defintion.IndexOn) + if defintion.Async { + args = append(args, "ASYNC") + } + if len(defintion.Prefix) > 0 { + args = append(args, "PREFIX", len(defintion.Prefix)) + for _, p := range defintion.Prefix { + args = append(args, p) + } + } + if defintion.FilterExpression != "" { + args = append(args, "FILTER", defintion.FilterExpression) + } + if defintion.Language != "" { + args = append(args, "LANGUAGE", defintion.Language) + } + + if defintion.LanguageField != "" { + args = append(args, "LANGUAGE_FIELD", defintion.LanguageField) + } + + if defintion.Score >= 0.0 && defintion.Score <= 1.0 { + args = append(args, "SCORE", defintion.Score) + } + + if defintion.ScoreField != "" { + args = append(args, "SCORE_FIELD", defintion.ScoreField) + } + if defintion.PayloadField != "" { + args = append(args, "PAYLOAD_FIELD", defintion.PayloadField) + } + return args +} + +func SerializeIndexingOptions(opts IndexingOptions, args redis.Args) redis.Args { + // apply options + + // As of RediSearch 2.0 and above NOSAVE is no longer supported. + if opts.NoSave { + args = append(args, "NOSAVE") + } + if opts.Language != "" { + args = append(args, "LANGUAGE", opts.Language) + } + + if opts.Partial { + opts.Replace = true + } + + if opts.Replace { + args = append(args, "REPLACE") + if opts.Partial { + args = append(args, "PARTIAL") + } + if opts.ReplaceCondition != "" { + args = append(args, "IF", opts.ReplaceCondition) + } + } + return args +} diff --git a/redisearch/index_test.go b/redisearch/index_test.go new file mode 100644 index 0000000..f2e42e2 --- /dev/null +++ b/redisearch/index_test.go @@ -0,0 +1,40 @@ +package redisearch + +import ( + "github.com/gomodule/redigo/redis" + "reflect" + "testing" +) + +func TestIndexDefinition_Serialize(t *testing.T) { + type fields struct { + Definition *IndexDefinition + } + type args struct { + args redis.Args + } + tests := []struct { + name string + fields fields + args args + want redis.Args + }{ + {"default", fields{NewIndexDefinition()}, args{redis.Args{}}, redis.Args{"ON", "HASH"}}, + {"default+async", fields{NewIndexDefinition().SetAsync(true)}, args{redis.Args{}}, redis.Args{"ON", "HASH", "ASYNC"}}, + {"default+score", fields{NewIndexDefinition().SetScore(0.75)}, args{redis.Args{}}, redis.Args{"ON", "HASH", "SCORE", 0.75}}, + {"default+score_field", fields{NewIndexDefinition().SetScoreField("myscore")}, args{redis.Args{}}, redis.Args{"ON", "HASH", "SCORE_FIELD", "myscore"}}, + {"default+language", fields{NewIndexDefinition().SetLanguage("portuguese")}, args{redis.Args{}}, redis.Args{"ON", "HASH", "LANGUAGE", "portuguese"}}, + {"default+language_field", fields{NewIndexDefinition().SetLanguageField("mylanguage")}, args{redis.Args{}}, redis.Args{"ON", "HASH", "LANGUAGE_FIELD", "mylanguage"}}, + {"default+prefix", fields{NewIndexDefinition().AddPrefix("products:*")}, args{redis.Args{}}, redis.Args{"ON", "HASH", "PREFIX", 1, "products:*"}}, + {"default+payload_field", fields{NewIndexDefinition().SetPayloadField("products_description")}, args{redis.Args{}}, redis.Args{"ON", "HASH", "PAYLOAD_FIELD", "products_description"}}, + {"default+filter", fields{NewIndexDefinition().SetFilterExpression("@score:[0 50]")}, args{redis.Args{}}, redis.Args{"ON", "HASH", "FILTER", "@score:[0 50]"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defintion := tt.fields.Definition + if got := defintion.Serialize(tt.args.args); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Serialize() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/redisearch/query.go b/redisearch/query.go index 3466ed1..54e8f6a 100644 --- a/redisearch/query.go +++ b/redisearch/query.go @@ -406,46 +406,3 @@ func (i *Client) IndexOptions(opts IndexingOptions, docs ...Document) error { return merr } - -func SerializeIndexingOptions(opts IndexingOptions, args redis.Args) redis.Args { - // apply options - if opts.NoSave { - args = append(args, "NOSAVE") - } - if opts.Language != "" { - args = append(args, "LANGUAGE", opts.Language) - } - - if opts.Partial { - opts.Replace = true - } - - if opts.Replace { - args = append(args, "REPLACE") - if opts.Partial { - args = append(args, "PARTIAL") - } - if opts.ReplaceCondition != "" { - args = append(args, "IF", opts.ReplaceCondition) - } - } - return args -} - -// IndexInfo - Structure showing information about an existing index -type IndexInfo struct { - Schema Schema - Name string `redis:"index_name"` - DocCount uint64 `redis:"num_docs"` - RecordCount uint64 `redis:"num_records"` - TermCount uint64 `redis:"num_terms"` - MaxDocID uint64 `redis:"max_doc_id"` - InvertedIndexSizeMB float64 `redis:"inverted_sz_mb"` - OffsetVectorSizeMB float64 `redis:"offset_vector_sz_mb"` - DocTableSizeMB float64 `redis:"doc_table_size_mb"` - KeyTableSizeMB float64 `redis:"key_table_size_mb"` - RecordsPerDocAvg float64 `redis:"records_per_doc_avg"` - BytesPerRecordAvg float64 `redis:"bytes_per_record_avg"` - OffsetsPerTermAvg float64 `redis:"offsets_per_term_avg"` - OffsetBitsPerTermAvg float64 `redis:"offset_bits_per_record_avg"` -} diff --git a/redisearch/query_test.go b/redisearch/query_test.go index ff57437..bc5151a 100644 --- a/redisearch/query_test.go +++ b/redisearch/query_test.go @@ -50,6 +50,7 @@ func Test_serializeIndexingOptions(t *testing.T) { }{ {"default with args", args{DefaultIndexingOptions, redis.Args{"idx1", "doc1", 1.0}}, redis.Args{"idx1", "doc1", 1.0}}, {"default", args{DefaultIndexingOptions, redis.Args{}}, redis.Args{}}, + {"default + language", args{IndexingOptions{Language: "portuguese"}, redis.Args{}}, redis.Args{"LANGUAGE", "portuguese"}}, {"replace full doc", args{IndexingOptions{Replace: true}, redis.Args{}}, redis.Args{"REPLACE"}}, {"replace partial", args{IndexingOptions{Replace: true, Partial: true}, redis.Args{}}, redis.Args{"REPLACE", "PARTIAL"}}, {"replace if", args{IndexingOptions{Replace: true, ReplaceCondition: "@timestamp < 23323234234"}, redis.Args{}}, redis.Args{"REPLACE", "IF", "@timestamp < 23323234234"}}, diff --git a/redisearch/redisearch_test.go b/redisearch/redisearch_test.go index a3e0787..10dcc92 100644 --- a/redisearch/redisearch_test.go +++ b/redisearch/redisearch_test.go @@ -92,16 +92,21 @@ func TestClient(t *testing.T) { } else { assert.Equal(t, 100, len(merr)) assert.NotEmpty(t, merr) - //fmt.Println("Got errors: ", merr) } } - docs, total, err := c.Search(NewQuery("hello world")) + // Wait for all documents to be indexed + info, _ := c.Info() + for info.IsIndexing { + time.Sleep(time.Second) + info, _ = c.Info() + } + docs, total, err := c.Search(NewQuery("hello world")) assert.Nil(t, err) assert.Equal(t, 100, total) assert.Equal(t, 10, len(docs)) - + teardown(c) } func TestInfo(t *testing.T) { @@ -116,6 +121,7 @@ func TestInfo(t *testing.T) { _, err := c.Info() assert.Nil(t, err) + teardown(c) } func TestNumeric(t *testing.T) { @@ -164,6 +170,7 @@ func TestNumeric(t *testing.T) { explain, err := c.Explain(NewQuery("hello world @bar:[40 90]")) assert.Nil(t, err) assert.NotNil(t, explain) + teardown(c) } func TestNoIndex(t *testing.T) { @@ -203,6 +210,7 @@ func TestNoIndex(t *testing.T) { docs, total, err = c.Search(NewQuery("@f2:Mark*").SetSortBy("f1", true)) assert.Equal(t, 2, total) assert.Equal(t, "TestNoIndex-doc2", docs[0].Id) + teardown(c) } func TestHighlight(t *testing.T) { @@ -253,6 +261,7 @@ func TestHighlight(t *testing.T) { } c.Drop() + teardown(c) } func TestSummarize(t *testing.T) { @@ -298,6 +307,7 @@ func TestSummarize(t *testing.T) { assert.Equal(t, "are two sub-[commands] [commands] used for highlighting. One is\r\na [field] into contextual [fragments] surrounding the found terms. It is possible to summarize a [field], highlight a [field], or\r\n", d.Properties["foo"]) assert.Equal(t, "hello world foo bar baz", d.Properties["bar"]) } + teardown(c) } func TestTags(t *testing.T) { @@ -345,7 +355,7 @@ func TestTags(t *testing.T) { assertNumResults("@tags:{hello world}", 1) assertNumResults("@tags:{hello world} @tags2:{foo\\ bar\\;bar}", 1) assertNumResults("hello world", 1) - + teardown(c) } func TestDelete(t *testing.T) { @@ -363,7 +373,9 @@ func TestDelete(t *testing.T) { // validate that the index is empty info, err = c.Info() assert.Nil(t, err) - assert.Equal(t, uint64(0), info.DocCount) + if !info.IsIndexing { + assert.Equal(t, uint64(0), info.DocCount) + } doc := NewDocument("TestDelete-doc1", 1.0) doc.Set("foo", "Hello world") @@ -371,6 +383,15 @@ func TestDelete(t *testing.T) { err = c.IndexOptions(DefaultIndexingOptions, doc) assert.Nil(t, err) + // Wait for all documents to be indexed + info, err = c.Info() + assert.Nil(t, err) + for info.IsIndexing { + time.Sleep(time.Second) + info, err = c.Info() + assert.Nil(t, err) + } + // now we should have 1 document (id = doc1) info, err = c.Info() assert.Nil(t, err) @@ -380,10 +401,18 @@ func TestDelete(t *testing.T) { err = c.Delete("TestDelete-doc1", true) assert.Nil(t, err) - // validate that the index is empty again + // Wait for all documents to be indexed info, err = c.Info() + assert.Nil(t, err) + for info.IsIndexing { + time.Sleep(time.Second) + info, err = c.Info() + assert.Nil(t, err) + } + assert.Nil(t, err) assert.Equal(t, uint64(0), info.DocCount) + teardown(c) } func TestSpellCheck(t *testing.T) { @@ -425,7 +454,7 @@ func TestSpellCheck(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 1, len(sugs)) assert.Equal(t, 1, total) - + teardown(c) } func TestFilter(t *testing.T) { @@ -434,7 +463,7 @@ func TestFilter(t *testing.T) { sc := NewSchema(DefaultOptions). AddField(NewTextField("body")). AddField(NewTextFieldOptions("title", TextFieldOptions{Weight: 5.0, Sortable: true})). - AddField(NewNumericField("age")). + AddField(NewNumericFieldOptions("age", NumericFieldOptions{Sortable: true})). AddField(NewGeoFieldOptions("location", GeoFieldOptions{})) c.Drop() @@ -473,4 +502,5 @@ func TestFilter(t *testing.T) { SetReturnFields("body")) assert.Nil(t, err) assert.Equal(t, 0, total) + teardown(c) }