Skip to content

Commit

Permalink
add onlyid=true to query endpoint
Browse files Browse the repository at this point in the history
  • Loading branch information
DocSavage committed Nov 18, 2024
1 parent 13e16b9 commit 66c7e5d
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 16 deletions.
5 changes: 1 addition & 4 deletions datatype/neuronjson/fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,7 @@ func removeReservedFields(data NeuronJSON, showFields Fields) NeuronJSON {
return out
}

// Return a subset of fields where
//
// onlyFields is a map of field names to include
// hideSuffixes is a map of fields suffixes (e.g., "_user") to exclude
// Return a subset of fields from the NeuronJSON
func selectFields(data NeuronJSON, fieldMap map[string]struct{}, showUser, showTime bool) NeuronJSON {
out := data.copy()
if len(fieldMap) > 0 {
Expand Down
38 changes: 26 additions & 12 deletions datatype/neuronjson/query.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ func queryMatch(queryList ListQueryJSON, value map[string]interface{}) (matches
return false, nil
}

func (d *Data) queryInMemory(mdb *memdb, w http.ResponseWriter, queryL ListQueryJSON, fieldMap map[string]struct{}, showFields Fields) (err error) {
func (d *Data) queryInMemory(mdb *memdb, w http.ResponseWriter, queryL ListQueryJSON, fieldMap map[string]struct{}, showFields Fields, onlyidOut bool) (err error) {
mdb.mu.RLock()
defer mdb.mu.RUnlock()

Expand All @@ -359,13 +359,18 @@ func (d *Data) queryInMemory(mdb *memdb, w http.ResponseWriter, queryL ListQuery
if matches, err = queryMatch(queryL, value); err != nil {
return
} else if matches {
if numMatches > 0 {
fmt.Fprint(w, ",")
}
if onlyidOut {
fmt.Fprintf(w, "%d", bodyid)
numMatches++
continue
}
out := selectFields(value, fieldMap, showUser, showTime)
if jsonBytes, err = json.Marshal(out); err != nil {
break
}
if numMatches > 0 {
fmt.Fprint(w, ",")
}
fmt.Fprint(w, string(jsonBytes))
numMatches++
}
Expand All @@ -374,35 +379,44 @@ func (d *Data) queryInMemory(mdb *memdb, w http.ResponseWriter, queryL ListQuery
}

func (d *Data) queryBackingStore(ctx storage.VersionedCtx, w http.ResponseWriter,
queryL ListQueryJSON, fieldMap map[string]struct{}, showFields Fields) (err error) {
queryL ListQueryJSON, fieldMap map[string]struct{}, showFields Fields, onlyidOut bool) (err error) {

dvid.Infof("store query using mdb with queryL: %v\n", queryL)
numMatches := 0
process_func := func(key string, value NeuronJSON) {
value = NeuronJSON(value)
if matches, err := queryMatch(queryL, value); err != nil {
dvid.Errorf("error in matching process: %v\n", err) // TODO: alter d.processRange to allow return of err
return
} else if !matches {
return
}
bodyidVal, ok := value["bodyid"]
if !ok {
dvid.Errorf("No bodyid found in annotation: %v", value)
return
}
if numMatches > 0 {
fmt.Fprint(w, ",")
}
if onlyidOut {
fmt.Fprint(w, bodyidVal)
numMatches++
return
}
out := removeReservedFields(value, showFields)
jsonBytes, err := json.Marshal(out)
if err != nil {
dvid.Errorf("error in JSON encoding: %v\n", err)
return
}
if numMatches > 0 {
fmt.Fprint(w, ",")
}
fmt.Fprint(w, string(jsonBytes))
numMatches++
}
return d.processStoreRange(ctx, process_func)
}

// Query reads POSTed data and returns JSON.
func (d *Data) Query(ctx *datastore.VersionedCtx, w http.ResponseWriter, uuid dvid.UUID, onlyid bool, fieldMap map[string]struct{}, showFields Fields, in io.ReadCloser) (err error) {
func (d *Data) Query(ctx *datastore.VersionedCtx, w http.ResponseWriter, uuid dvid.UUID, onlyidOut bool, fieldMap map[string]struct{}, showFields Fields, in io.ReadCloser) (err error) {
var queryBytes []byte
if queryBytes, err = io.ReadAll(in); err != nil {
return
Expand Down Expand Up @@ -434,11 +448,11 @@ func (d *Data) Query(ctx *datastore.VersionedCtx, w http.ResponseWriter, uuid dv
fmt.Fprint(w, "[")
mdb, found := d.getMemDBbyVersion(ctx.VersionID())
if found {
if err = d.queryInMemory(mdb, w, queryL, fieldMap, showFields); err != nil {
if err = d.queryInMemory(mdb, w, queryL, fieldMap, showFields, onlyidOut); err != nil {
return
}
} else {
if err = d.queryBackingStore(ctx, w, queryL, fieldMap, showFields); err != nil {
if err = d.queryBackingStore(ctx, w, queryL, fieldMap, showFields, onlyidOut); err != nil {
return
}
}
Expand Down
25 changes: 25 additions & 0 deletions datatype/neuronjson/query_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"strings"
"testing"

"github.com/janelia-flyem/dvid/datastore"
"github.com/janelia-flyem/dvid/server"
)

Expand Down Expand Up @@ -73,4 +74,28 @@ func TestQueryBodyIDs(t *testing.T) {
if !equalListJSON(returnValue, expectedValue, ShowBasic) {
t.Fatalf("Bad query request return. Expected:%v. Got: %v\n", string(expectedValue), string(returnValue))
}

// Test if we use bodyid=true query string
query = `{"class": "9A"}`
queryreq = fmt.Sprintf("%snode/%s/neurons/query?onlyid=true", server.WebAPIPath, uuid)
returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))

if string(returnValue) != "[2000,2001,2002,2003]" {
t.Fatalf("Bad query request return.\nExpected: [2000,2001,2002,2003]\nGot: %v\n", string(returnValue))
}

// Commit current version and make new child to test querying on backing store instead of memory.
if err := datastore.Commit(uuid, "my commit msg", []string{"stuff one", "stuff two"}); err != nil {
t.Fatalf("Unable to lock root node %s: %v\n", uuid, err)
}
_, err := datastore.NewVersion(uuid, "some child", "", nil)
if err != nil {
t.Fatalf("Unable to create new version off node %s: %v\n", uuid, err)
}
queryreq = fmt.Sprintf("%snode/%s/neurons/query?onlyid=true", server.WebAPIPath, uuid)
returnValue = server.TestHTTP(t, "POST", queryreq, strings.NewReader(query))

if string(returnValue) != "[2000,2001,2002,2003]" {
t.Fatalf("Bad query request return.\nExpected: [2000,2001,2002,2003]\nGot: %v\n", string(returnValue))
}
}

0 comments on commit 66c7e5d

Please sign in to comment.