Merge pull request #3635 from aduffeck/search

Add initial version of the search extension

changelog/unreleased/search-extension.md

@@ -0,0 +1,5 @@
+Enhancement: Add initial version of the search extensions
+
+It is now possible to search for files and directories by their name using the web UI. Therefor new search extension indexes files in a persistent local index.
+
+https://github.com/owncloud/ocis/pull/3635

docs/extensions/port-ranges.md

@@ -42,7 +42,7 @@ We also suggest to use the last port in your extensions' range as a debug/metric
 | 9205-9209  | [markdown-editor](https://github.com/owncloud/ocis-markdown-editor)           |
 | 9210-9214  | [reva](https://github.com/owncloud/ocis-reva) unused?                         |
 | 9215-9219  | reva metadata storage                                                         |
-| 9220-9224  | FREE                                                                          |
+| 9220-9224  | search                                                                          |
 | 9225-9229  | photoprism (state: PoC)                                                       |
 | 9230-9234  | [nats](https://github.com/owncloud/ocis/tree/master/nats)                     |
 | 9235-9239  | idm TBD                                                                       |

docs/extensions/storage/ports.md

@@ -34,10 +34,13 @@ For now, the storage service uses these ports to preconfigure those services:
 | 9159      | storage users debug                           |
 | 9160      | groups                                        |
 | 9161      | groups debug                                  |
-| 9164      | storage appprovider                                   |
-| 9165      | storage appprovider debug                             |
+| 9164      | storage appprovider                           |
+| 9165      | storage appprovider debug                     |
 | 9178      | storage public link                           |
 | 9179      | storage public link data                      |
+| 9180      | accounts grpc                                 |
+| 9181      | accounts http                                 |
+| 9182      | accounts debug                                |
 | 9215      | storage meta grpc                             |
 | 9216      | storage meta http                             |
 | 9217      | storage meta debug                            |

extensions/audit/pkg/types/events.go

@@ -16,6 +16,7 @@ func RegisteredEvents() []events.Unmarshaller {
 		events.ReceivedShareUpdated{},
 		events.LinkAccessed{},
 		events.LinkAccessFailed{},
+		events.ContainerCreated{},
 		events.FileUploaded{},
 		events.FileDownloaded{},
 		events.ItemTrashed{},

extensions/frontend/pkg/command/command.go

@@ -236,7 +236,9 @@ func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[s
 								"preferred_upload_type": cfg.Checksums.PreferredUploadType,
 							},
 							"files": filesCfg,
-							"dav":   map[string]interface{}{},
+							"dav": map[string]interface{}{
+								"reports": []string{"search-files"},
+							},
 							"files_sharing": map[string]interface{}{
 								"api_enabled":                       true,
 								"resharing":                         false,

extensions/proxy/pkg/config/config.go

@@ -45,8 +45,10 @@ type Policy struct {
 
 // Route defines forwarding routes
 type Route struct {
-	Type     RouteType `yaml:"type"`
-	Endpoint string    `yaml:"endpoint"`
+	Type RouteType `yaml:"type"`
+	// Method optionally limits the route to this HTTP method
+	Method   string `yaml:"method"`
+	Endpoint string `yaml:"endpoint"`
 	// Backend is a static URL to forward the request to
 	Backend string `yaml:"backend"`
 	// Service name to look up in the registry

extensions/proxy/pkg/config/defaults/defaultconfig.go

@@ -97,6 +97,15 @@ func DefaultPolicies() []config.Policy {
 					Endpoint: "/remote.php/?preview=1",
 					Backend:  "http://localhost:9115",
 				},
+				{
+					// TODO the actual REPORT goes to /dav/files/{username}, which is user specific ... how would this work in a spaces world?
+					// TODO what paths are returned? the href contains the full path so it should be possible to return urls from other spaces?
+					// TODO or we allow a REPORT on /dav/spaces to search all spaces and /dav/space/{spaceid} to search a specific space
+					// send webdav REPORT requests to search service
+					Method:   "REPORT",
+					Endpoint: "/remote.php/dav/",
+					Backend:  "http://localhost:9115", // TODO use registry?
+				},
 				{
 					Endpoint: "/remote.php/",
 					Service:  "ocdav",

extensions/proxy/pkg/proxy/proxy.go

@@ -29,7 +29,8 @@ import (
 // MultiHostReverseProxy extends "httputil" to support multiple hosts with different policies
 type MultiHostReverseProxy struct {
 	httputil.ReverseProxy
-	Directors      map[string]map[config.RouteType]map[string]func(req *http.Request)
+	// Directors holds policy        route type        method    endpoint         Director
+	Directors      map[string]map[config.RouteType]map[string]map[string]func(req *http.Request)
 	PolicySelector policy.Selector
 	logger         log.Logger
 	config         *config.Config
@@ -40,7 +41,7 @@ func NewMultiHostReverseProxy(opts ...Option) *MultiHostReverseProxy {
 	options := newOptions(opts...)
 
 	rp := &MultiHostReverseProxy{
-		Directors: make(map[string]map[config.RouteType]map[string]func(req *http.Request)),
+		Directors: make(map[string]map[config.RouteType]map[string]map[string]func(req *http.Request)),
 		logger:    options.Logger,
 		config:    options.Config,
 	}
@@ -124,6 +125,7 @@ func (p *MultiHostReverseProxy) directorSelectionDirector(r *http.Request) {
 		return
 	}
 
+	method := ""
 	// find matching director
 	for _, rt := range config.RouteTypes {
 		var handler func(string, url.URL) bool
@@ -137,25 +139,36 @@ func (p *MultiHostReverseProxy) directorSelectionDirector(r *http.Request) {
 		default:
 			handler = p.prefixRouteMatcher
 		}
-		for endpoint := range p.Directors[pol][rt] {
+		if p.Directors[pol][rt][r.Method] != nil {
+			// use specific method
+			method = r.Method
+		}
+		for endpoint := range p.Directors[pol][rt][method] {
 			if handler(endpoint, *r.URL) {
 
 				p.logger.Debug().
 					Str("policy", pol).
+					Str("method", r.Method).
 					Str("prefix", endpoint).
 					Str("path", r.URL.Path).
 					Str("routeType", string(rt)).
 					Msg("director found")
 
-				p.Directors[pol][rt][endpoint](r)
+				p.Directors[pol][rt][method][endpoint](r)
 				return
 			}
 		}
 	}
 
 	// override default director with root. If any
-	if p.Directors[pol][config.PrefixRoute]["/"] != nil {
-		p.Directors[pol][config.PrefixRoute]["/"](r)
+	switch {
+	case p.Directors[pol][config.PrefixRoute][method]["/"] != nil:
+		// try specific method
+		p.Directors[pol][config.PrefixRoute][method]["/"](r)
+		return
+	case p.Directors[pol][config.PrefixRoute][""]["/"] != nil:
+		// fallback to unspecific method
+		p.Directors[pol][config.PrefixRoute][""]["/"](r)
 		return
 	}
 
@@ -182,20 +195,23 @@ func singleJoiningSlash(a, b string) string {
 func (p *MultiHostReverseProxy) AddHost(policy string, target *url.URL, rt config.Route) {
 	targetQuery := target.RawQuery
 	if p.Directors[policy] == nil {
-		p.Directors[policy] = make(map[config.RouteType]map[string]func(req *http.Request))
+		p.Directors[policy] = make(map[config.RouteType]map[string]map[string]func(req *http.Request))
 	}
 	routeType := config.DefaultRouteType
 	if rt.Type != "" {
 		routeType = rt.Type
 	}
 	if p.Directors[policy][routeType] == nil {
-		p.Directors[policy][routeType] = make(map[string]func(req *http.Request))
+		p.Directors[policy][routeType] = make(map[string]map[string]func(req *http.Request))
+	}
+	if p.Directors[policy][routeType][rt.Method] == nil {
+		p.Directors[policy][routeType][rt.Method] = make(map[string]func(req *http.Request))
 	}
 
 	reg := registry.GetRegistry()
 	sel := selector.NewSelector(selector.Registry(reg))
 
-	p.Directors[policy][routeType][rt.Endpoint] = func(req *http.Request) {
+	p.Directors[policy][routeType][rt.Method][rt.Endpoint] = func(req *http.Request) {
 		if rt.Service != "" {
 			// select next node
 			next, err := sel.Select(rt.Service)

extensions/proxy/pkg/proxy/proxy_test.go

@@ -1,15 +1,19 @@
 package proxy
 
 import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"testing"
 
+	"github.com/owncloud/ocis/extensions/proxy/pkg/config"
 	"github.com/owncloud/ocis/extensions/proxy/pkg/config/defaults"
 )
 
 type matchertest struct {
-	endpoint, target string
-	matches          bool
+	method, endpoint, target string
+	matches                  bool
 }
 
 func TestPrefixRouteMatcher(t *testing.T) {
@@ -99,3 +103,35 @@ func TestSingleJoiningSlash(t *testing.T) {
 		}
 	}
 }
+
+func TestDirectorSelectionDirector(t *testing.T) {
+
+	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "ok")
+	}))
+	defer svr.Close()
+
+	p := NewMultiHostReverseProxy(Config(&config.Config{
+		PolicySelector: &config.PolicySelector{
+			Static: &config.StaticSelectorConf{
+				Policy: "default",
+			},
+		},
+	}))
+	p.AddHost("default", &url.URL{Host: "ocdav"}, config.Route{Type: config.PrefixRoute, Method: "", Endpoint: "/dav", Backend: "ocdav"})
+	p.AddHost("default", &url.URL{Host: "ocis-webdav"}, config.Route{Type: config.PrefixRoute, Method: "REPORT", Endpoint: "/dav", Backend: "ocis-webdav"})
+
+	table := []matchertest{
+		{method: "PROPFIND", endpoint: "/dav/files/demo/", target: "ocdav"},
+		{method: "REPORT", endpoint: "/dav/files/demo/", target: "ocis-webdav"},
+	}
+
+	for _, test := range table {
+		r := httptest.NewRequest(test.method, "/dav/files/demo/", nil)
+		p.directorSelectionDirector(r)
+		if r.URL.Host != test.target {
+			t.Errorf("TestDirectorSelectionDirector got host %s expected %s", r.Host, test.target)
+
+		}
+	}
+}

extensions/search/cmd/search/main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"os"
+
+	"github.com/owncloud/ocis/extensions/search/pkg/command"
+	"github.com/owncloud/ocis/extensions/search/pkg/config/defaults"
+)
+
+func main() {
+	if err := command.Execute(defaults.DefaultConfig()); err != nil {
+		os.Exit(1)
+	}
+}

extensions/search/pkg/command/health.go

@@ -0,0 +1,53 @@
+package command
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/owncloud/ocis/extensions/search/pkg/config"
+	"github.com/owncloud/ocis/extensions/search/pkg/config/parser"
+	"github.com/owncloud/ocis/extensions/search/pkg/logging"
+	"github.com/urfave/cli/v2"
+)
+
+// Health is the entrypoint for the health command.
+func Health(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:     "health",
+		Usage:    "check health status",
+		Category: "info",
+		Before: func(c *cli.Context) error {
+			return parser.ParseConfig(cfg)
+		},
+		Action: func(c *cli.Context) error {
+			logger := logging.Configure(cfg.Service.Name, cfg.Log)
+
+			resp, err := http.Get(
+				fmt.Sprintf(
+					"http://%s/healthz",
+					cfg.Debug.Addr,
+				),
+			)
+
+			if err != nil {
+				logger.Fatal().
+					Err(err).
+					Msg("Failed to request health check")
+			}
+
+			defer resp.Body.Close()
+
+			if resp.StatusCode != http.StatusOK {
+				logger.Fatal().
+					Int("code", resp.StatusCode).
+					Msg("Health seems to be in bad state")
+			}
+
+			logger.Debug().
+				Int("code", resp.StatusCode).
+				Msg("Health got a good state")
+
+			return nil
+		},
+	}
+}

extensions/search/pkg/command/index.go

@@ -0,0 +1,52 @@
+package command
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/owncloud/ocis/extensions/search/pkg/config"
+	"github.com/owncloud/ocis/extensions/search/pkg/config/parser"
+	"github.com/owncloud/ocis/ocis-pkg/service/grpc"
+	searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0"
+)
+
+// Index is the entrypoint for the server command.
+func Index(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:     "index",
+		Usage:    "index the files for one one more users",
+		Category: "index management",
+		Aliases:  []string{"i"},
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "space",
+				Aliases:  []string{"s"},
+				Required: true,
+				Usage:    "the id of the space to travers and index the files of",
+			},
+			&cli.StringFlag{
+				Name:     "user",
+				Aliases:  []string{"u"},
+				Required: true,
+				Usage:    "the username of the user tha shall be used to access the files",
+			},
+		},
+		Before: func(c *cli.Context) error {
+			return parser.ParseConfig(cfg)
+		},
+		Action: func(c *cli.Context) error {
+			client := searchsvc.NewSearchProviderService("com.owncloud.api.search", grpc.DefaultClient)
+			_, err := client.IndexSpace(context.Background(), &searchsvc.IndexSpaceRequest{
+				SpaceId: c.String("space"),
+				UserId:  c.String("user"),
+			})
+			if err != nil {
+				fmt.Println("failed to index space: " + err.Error())
+				return err
+			}
+			return nil
+		},
+	}
+}