Externalize custom mime types configuration for storage providers (#2613

)

changelog/unreleased/mimetypes-config.md

@@ -0,0 +1,5 @@
+Enhancement: Externalize custom mime types configuration for storage providers
+
+Added ability to configure custom mime types in an external JSON file, such that it can be reused when many storage providers are deployed at the same time.
+
+https://github.com/cs3org/reva/pull/2613

docs/content/en/docs/config/grpc/services/storageprovider/_index.md

@@ -9,31 +9,31 @@ description: >
 # _struct: config_
 
 {{% dir name="mount_path" type="string" default="/" %}}
-The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L56)
+The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 mount_path = "/"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="mount_id" type="string" default="-" %}}
-The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L57)
+The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 mount_id = "-"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="driver" type="string" default="localhome" %}}
-The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58)
+The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 driver = "localhome"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="drivers" type="map[string]map[string]interface{}" default="localhome" %}}
- [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59)
+ [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61)
 {{< highlight toml >}}
 [grpc.services.storageprovider.drivers.localhome]
 root = "/var/tmp/reva/"
@@ -44,42 +44,42 @@ user_layout = "{{.Username}}"
 {{% /dir %}}
 
 {{% dir name="tmp_folder" type="string" default="/var/tmp" %}}
-Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60)
+Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 tmp_folder = "/var/tmp"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="data_server_url" type="string" default="http://localhost/data" %}}
-The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61)
+The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 data_server_url = "http://localhost/data"
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="expose_data_server" type="bool" default=false %}}
-Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62)
+Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 expose_data_server = false
 {{< /highlight >}}
 {{% /dir %}}
 
 {{% dir name="available_checksums" type="map[string]uint32" default=nil %}}
-List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63)
+List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
 available_checksums = nil
 {{< /highlight >}}
 {{% /dir %}}
 
-{{% dir name="mimetypes" type="map[string]string" default=nil %}}
-List of supported mime types and corresponding file extensions. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64)
+{{% dir name="custom_mimetypes_json" type="string" default="nil" %}}
+An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66)
 {{< highlight toml >}}
 [grpc.services.storageprovider]
-mimetypes = nil
+custom_mimetypes_json = "nil"
 {{< /highlight >}}
 {{% /dir %}}
 

examples/nextcloud-integration/custom-mime-types-demo.json

@@ -0,0 +1,4 @@
+{
+    ".zmd": "application/compressed-markdown",
+    ".zep": "application/compressed-etherpad"
+}

examples/nextcloud-integration/revad.toml

@@ -90,9 +90,7 @@ expose_data_server = true
 data_server_url = "http://127.0.0.1:19001/data"
 enable_home_creation = true
 disable_tus = true
-
-[grpc.services.storageprovider.mimetypes]
-".zmd" = "application/compressed-markdown"
+custom_mime_types_json = "custom-mime-types-demo.json"
 
 [grpc.services.storageprovider.drivers.nextcloud]
 end_point = "http://localhost/apps/sciencemesh/"

examples/ocmd/ocmd-server-1.toml

@@ -97,9 +97,6 @@ expose_data_server = true
 data_server_url = "http://localhost:19001/data"
 enable_home_creation = true
 
-[grpc.services.storageprovider.mimetypes]
-".zmd" = "application/compressed-markdown"
-
 [grpc.services.storageprovider.drivers.localhome]
 user_layout = "{{.Username}}"
 

internal/grpc/services/storageprovider/storageprovider.go

@@ -20,7 +20,9 @@ package storageprovider
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path"
@@ -53,15 +55,15 @@ func init() {
 }
 
 type config struct {
-	MountPath        string                            `mapstructure:"mount_path" docs:"/;The path where the file system would be mounted."`
-	MountID          string                            `mapstructure:"mount_id" docs:"-;The ID of the mounted file system."`
-	Driver           string                            `mapstructure:"driver" docs:"localhome;The storage driver to be used."`
-	Drivers          map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go"`
-	TmpFolder        string                            `mapstructure:"tmp_folder" docs:"/var/tmp;Path to temporary folder."`
-	DataServerURL    string                            `mapstructure:"data_server_url" docs:"http://localhost/data;The URL for the data server."`
-	ExposeDataServer bool                              `mapstructure:"expose_data_server" docs:"false;Whether to expose data server."` // if true the client will be able to upload/download directly to it
-	AvailableXS      map[string]uint32                 `mapstructure:"available_checksums" docs:"nil;List of available checksums."`
-	MimeTypes        map[string]string                 `mapstructure:"mimetypes" docs:"nil;List of supported mime types and corresponding file extensions."`
+	MountPath           string                            `mapstructure:"mount_path" docs:"/;The path where the file system would be mounted."`
+	MountID             string                            `mapstructure:"mount_id" docs:"-;The ID of the mounted file system."`
+	Driver              string                            `mapstructure:"driver" docs:"localhome;The storage driver to be used."`
+	Drivers             map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go"`
+	TmpFolder           string                            `mapstructure:"tmp_folder" docs:"/var/tmp;Path to temporary folder."`
+	DataServerURL       string                            `mapstructure:"data_server_url" docs:"http://localhost/data;The URL for the data server."`
+	ExposeDataServer    bool                              `mapstructure:"expose_data_server" docs:"false;Whether to expose data server."` // if true the client will be able to upload/download directly to it
+	AvailableXS         map[string]uint32                 `mapstructure:"available_checksums" docs:"nil;List of available checksums."`
+	CustomMimeTypesJSON string                            `mapstructure:"custom_mimetypes_json" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."`
 }
 
 func (c *config) init() {
@@ -94,10 +96,6 @@ func (c *config) init() {
 	if len(c.AvailableXS) == 0 {
 		c.AvailableXS = map[string]uint32{"md5": 100, "unset": 1000}
 	}
-	if c.MimeTypes == nil || len(c.MimeTypes) == 0 {
-		c.MimeTypes = map[string]string{".zmd": "application/compressed-markdown"}
-	}
-
 }
 
 type service struct {
@@ -144,6 +142,25 @@ func parseConfig(m map[string]interface{}) (*config, error) {
 	return c, nil
 }
 
+func registerMimeTypes(mappingFile string) error {
+	if mappingFile != "" {
+		f, err := ioutil.ReadFile(mappingFile)
+		if err != nil {
+			return fmt.Errorf("storageprovider: error reading the custom mime types file: +%v", err)
+		}
+		mimeTypes := map[string]string{}
+		err = json.Unmarshal(f, &mimeTypes)
+		if err != nil {
+			return fmt.Errorf("storageprovider: error unmarshalling the custom mime types file: +%v", err)
+		}
+		// register all mime types that were read
+		for e, m := range mimeTypes {
+			mime.RegisterMime(e, m)
+		}
+	}
+	return nil
+}
+
 // New creates a new storage provider svc
 func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 
@@ -182,7 +199,11 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 		return nil, errtypes.NotFound("no available checksum, please set in config")
 	}
 
-	registerMimeTypes(c.MimeTypes)
+	// read and register custom mime types if configured
+	err = registerMimeTypes(c.CustomMimeTypesJSON)
+	if err != nil {
+		return nil, err
+	}
 
 	service := &service{
 		conf:          c,
@@ -197,12 +218,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
 	return service, nil
 }
 
-func registerMimeTypes(mimes map[string]string) {
-	for k, v := range mimes {
-		mime.RegisterMime(k, v)
-	}
-}
-
 func (s *service) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitraryMetadataRequest) (*provider.SetArbitraryMetadataResponse, error) {
 	newRef, err := s.unwrap(ctx, req.Ref)
 	if err != nil {