From 9550f678e563d079821bf81d1e714e7e84d1af03 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 3 Mar 2022 15:01:46 +0100 Subject: [PATCH 1/2] Externalize custom mime types for storage providers --- changelog/unreleased/mimetypes-config.md | 5 ++ .../grpc/services/storageprovider/_index.md | 22 ++++---- .../custom-mime-types-demo.json | 10 ++++ examples/nextcloud-integration/revad.toml | 4 +- examples/ocmd/ocmd-server-1.toml | 3 -- .../storageprovider/storageprovider.go | 53 ++++++++++++++----- 6 files changed, 68 insertions(+), 29 deletions(-) create mode 100644 changelog/unreleased/mimetypes-config.md create mode 100644 examples/nextcloud-integration/custom-mime-types-demo.json diff --git a/changelog/unreleased/mimetypes-config.md b/changelog/unreleased/mimetypes-config.md new file mode 100644 index 0000000000..680115abaa --- /dev/null +++ b/changelog/unreleased/mimetypes-config.md @@ -0,0 +1,5 @@ +Enhancement: Externalize custom mime types configuration for storage providers + +Added ability to configure custom mime types in an external JSON file, such that it can be reused when many storage providers are deployed at the same time. + +https://github.com/cs3org/reva/pull/2613 diff --git a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md index 728c382f59..fbea4db9c4 100644 --- a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="mount_path" type="string" default="/" %}} -The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L56) +The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58) {{< highlight toml >}} [grpc.services.storageprovider] mount_path = "/" @@ -17,7 +17,7 @@ mount_path = "/" {{% /dir %}} {{% dir name="mount_id" type="string" default="-" %}} -The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L57) +The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59) {{< highlight toml >}} [grpc.services.storageprovider] mount_id = "-" @@ -25,7 +25,7 @@ mount_id = "-" {{% /dir %}} {{% dir name="driver" type="string" default="localhome" %}} -The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58) +The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60) {{< highlight toml >}} [grpc.services.storageprovider] driver = "localhome" @@ -33,7 +33,7 @@ driver = "localhome" {{% /dir %}} {{% dir name="drivers" type="map[string]map[string]interface{}" default="localhome" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61) {{< highlight toml >}} [grpc.services.storageprovider.drivers.localhome] root = "/var/tmp/reva/" @@ -44,7 +44,7 @@ user_layout = "{{.Username}}" {{% /dir %}} {{% dir name="tmp_folder" type="string" default="/var/tmp" %}} -Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60) +Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62) {{< highlight toml >}} [grpc.services.storageprovider] tmp_folder = "/var/tmp" @@ -52,7 +52,7 @@ tmp_folder = "/var/tmp" {{% /dir %}} {{% dir name="data_server_url" type="string" default="http://localhost/data" %}} -The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61) +The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) {{< highlight toml >}} [grpc.services.storageprovider] data_server_url = "http://localhost/data" @@ -60,7 +60,7 @@ data_server_url = "http://localhost/data" {{% /dir %}} {{% dir name="expose_data_server" type="bool" default=false %}} -Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62) +Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) {{< highlight toml >}} [grpc.services.storageprovider] expose_data_server = false @@ -68,18 +68,18 @@ expose_data_server = false {{% /dir %}} {{% dir name="available_checksums" type="map[string]uint32" default=nil %}} -List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) +List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65) {{< highlight toml >}} [grpc.services.storageprovider] available_checksums = nil {{< /highlight >}} {{% /dir %}} -{{% dir name="mimetypes" type="map[string]string" default=nil %}} -List of supported mime types and corresponding file extensions. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) +{{% dir name="custom_mimetypes" type="string" default="nil" %}} +An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66) {{< highlight toml >}} [grpc.services.storageprovider] -mimetypes = nil +custom_mimetypes = "nil" {{< /highlight >}} {{% /dir %}} diff --git a/examples/nextcloud-integration/custom-mime-types-demo.json b/examples/nextcloud-integration/custom-mime-types-demo.json new file mode 100644 index 0000000000..828af503d9 --- /dev/null +++ b/examples/nextcloud-integration/custom-mime-types-demo.json @@ -0,0 +1,10 @@ +[ + { + "file_ext": ".zmd", + "mime_type": "application/compressed-markdown" + }, + { + "file_ext": ".zep", + "mime_type": "application/compressed-etherpad" + } +] diff --git a/examples/nextcloud-integration/revad.toml b/examples/nextcloud-integration/revad.toml index 6c9b0c0615..77bf14be86 100644 --- a/examples/nextcloud-integration/revad.toml +++ b/examples/nextcloud-integration/revad.toml @@ -90,9 +90,7 @@ expose_data_server = true data_server_url = "http://127.0.0.1:19001/data" enable_home_creation = true disable_tus = true - -[grpc.services.storageprovider.mimetypes] -".zmd" = "application/compressed-markdown" +custom_mime_types = "custom-mime-types-demo.json" [grpc.services.storageprovider.drivers.nextcloud] end_point = "http://localhost/apps/sciencemesh/" diff --git a/examples/ocmd/ocmd-server-1.toml b/examples/ocmd/ocmd-server-1.toml index dad77ce087..2e41678afb 100644 --- a/examples/ocmd/ocmd-server-1.toml +++ b/examples/ocmd/ocmd-server-1.toml @@ -97,9 +97,6 @@ expose_data_server = true data_server_url = "http://localhost:19001/data" enable_home_creation = true -[grpc.services.storageprovider.mimetypes] -".zmd" = "application/compressed-markdown" - [grpc.services.storageprovider.drivers.localhome] user_layout = "{{.Username}}" diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index fd72dbcb7c..2b1fdd5f46 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -20,7 +20,9 @@ package storageprovider import ( "context" + "encoding/json" "fmt" + "io/ioutil" "net/url" "os" "path" @@ -61,7 +63,12 @@ type config struct { DataServerURL string `mapstructure:"data_server_url" docs:"http://localhost/data;The URL for the data server."` ExposeDataServer bool `mapstructure:"expose_data_server" docs:"false;Whether to expose data server."` // if true the client will be able to upload/download directly to it AvailableXS map[string]uint32 `mapstructure:"available_checksums" docs:"nil;List of available checksums."` - MimeTypes map[string]string `mapstructure:"mimetypes" docs:"nil;List of supported mime types and corresponding file extensions."` + CustomMimeTypes string `mapstructure:"custom_mimetypes" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."` +} + +type mimeType struct { + FileExt string `mapstructure:"file_ext" json:"file_ext"` + MimeType string `mapstructure:"mime_type" json:"mime_type"` } func (c *config) init() { @@ -94,10 +101,6 @@ func (c *config) init() { if len(c.AvailableXS) == 0 { c.AvailableXS = map[string]uint32{"md5": 100, "unset": 1000} } - if c.MimeTypes == nil || len(c.MimeTypes) == 0 { - c.MimeTypes = map[string]string{".zmd": "application/compressed-markdown"} - } - } type service struct { @@ -144,6 +147,34 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } +func registerMimeTypes(mappingFile string) error { + mimeTypes := map[string]string{} + + if mappingFile != "" { + f, err := ioutil.ReadFile(mappingFile) + if err != nil { + return fmt.Errorf("storageprovider: error reading the custom mime types file: +%v", err) + } + mimes := []*mimeType{} + err = json.Unmarshal(f, &mimes) + if err != nil { + return fmt.Errorf("storageprovider: error unmarshalling the custom mime types file: +%v", err) + } + for _, m := range mimes { + if _, found := mimeTypes[m.FileExt]; found { + return fmt.Errorf("storageprovider: mimetypes mapping error, file extension \"%s\" is mapped to multiple mime types", m.FileExt) + } + mimeTypes[m.FileExt] = m.MimeType + } + + // now register all mime types that were read + for k, v := range mimeTypes { + mime.RegisterMime(k, v) + } + } + return nil +} + // New creates a new storage provider svc func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { @@ -182,7 +213,11 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return nil, errtypes.NotFound("no available checksum, please set in config") } - registerMimeTypes(c.MimeTypes) + // read and register custom mime types if configured + err = registerMimeTypes(c.CustomMimeTypes) + if err != nil { + return nil, err + } service := &service{ conf: c, @@ -197,12 +232,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return service, nil } -func registerMimeTypes(mimes map[string]string) { - for k, v := range mimes { - mime.RegisterMime(k, v) - } -} - func (s *service) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitraryMetadataRequest) (*provider.SetArbitraryMetadataResponse, error) { newRef, err := s.unwrap(ctx, req.Ref) if err != nil { From b1ca216020f58d7dc79e98e9fa275db447c10c77 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Mon, 7 Mar 2022 14:42:31 +0100 Subject: [PATCH 2/2] Addressed review comments --- .../grpc/services/storageprovider/_index.md | 4 +- .../custom-mime-types-demo.json | 14 ++---- examples/nextcloud-integration/revad.toml | 2 +- .../storageprovider/storageprovider.go | 44 +++++++------------ 4 files changed, 22 insertions(+), 42 deletions(-) diff --git a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md index fbea4db9c4..0a9ff8c468 100644 --- a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md @@ -75,11 +75,11 @@ available_checksums = nil {{< /highlight >}} {{% /dir %}} -{{% dir name="custom_mimetypes" type="string" default="nil" %}} +{{% dir name="custom_mimetypes_json" type="string" default="nil" %}} An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66) {{< highlight toml >}} [grpc.services.storageprovider] -custom_mimetypes = "nil" +custom_mimetypes_json = "nil" {{< /highlight >}} {{% /dir %}} diff --git a/examples/nextcloud-integration/custom-mime-types-demo.json b/examples/nextcloud-integration/custom-mime-types-demo.json index 828af503d9..a48c986790 100644 --- a/examples/nextcloud-integration/custom-mime-types-demo.json +++ b/examples/nextcloud-integration/custom-mime-types-demo.json @@ -1,10 +1,4 @@ -[ - { - "file_ext": ".zmd", - "mime_type": "application/compressed-markdown" - }, - { - "file_ext": ".zep", - "mime_type": "application/compressed-etherpad" - } -] +{ + ".zmd": "application/compressed-markdown", + ".zep": "application/compressed-etherpad" +} diff --git a/examples/nextcloud-integration/revad.toml b/examples/nextcloud-integration/revad.toml index 77bf14be86..52913b88ad 100644 --- a/examples/nextcloud-integration/revad.toml +++ b/examples/nextcloud-integration/revad.toml @@ -90,7 +90,7 @@ expose_data_server = true data_server_url = "http://127.0.0.1:19001/data" enable_home_creation = true disable_tus = true -custom_mime_types = "custom-mime-types-demo.json" +custom_mime_types_json = "custom-mime-types-demo.json" [grpc.services.storageprovider.drivers.nextcloud] end_point = "http://localhost/apps/sciencemesh/" diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 2b1fdd5f46..a988678110 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -55,20 +55,15 @@ func init() { } type config struct { - MountPath string `mapstructure:"mount_path" docs:"/;The path where the file system would be mounted."` - MountID string `mapstructure:"mount_id" docs:"-;The ID of the mounted file system."` - Driver string `mapstructure:"driver" docs:"localhome;The storage driver to be used."` - Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go"` - TmpFolder string `mapstructure:"tmp_folder" docs:"/var/tmp;Path to temporary folder."` - DataServerURL string `mapstructure:"data_server_url" docs:"http://localhost/data;The URL for the data server."` - ExposeDataServer bool `mapstructure:"expose_data_server" docs:"false;Whether to expose data server."` // if true the client will be able to upload/download directly to it - AvailableXS map[string]uint32 `mapstructure:"available_checksums" docs:"nil;List of available checksums."` - CustomMimeTypes string `mapstructure:"custom_mimetypes" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."` -} - -type mimeType struct { - FileExt string `mapstructure:"file_ext" json:"file_ext"` - MimeType string `mapstructure:"mime_type" json:"mime_type"` + MountPath string `mapstructure:"mount_path" docs:"/;The path where the file system would be mounted."` + MountID string `mapstructure:"mount_id" docs:"-;The ID of the mounted file system."` + Driver string `mapstructure:"driver" docs:"localhome;The storage driver to be used."` + Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go"` + TmpFolder string `mapstructure:"tmp_folder" docs:"/var/tmp;Path to temporary folder."` + DataServerURL string `mapstructure:"data_server_url" docs:"http://localhost/data;The URL for the data server."` + ExposeDataServer bool `mapstructure:"expose_data_server" docs:"false;Whether to expose data server."` // if true the client will be able to upload/download directly to it + AvailableXS map[string]uint32 `mapstructure:"available_checksums" docs:"nil;List of available checksums."` + CustomMimeTypesJSON string `mapstructure:"custom_mimetypes_json" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."` } func (c *config) init() { @@ -148,28 +143,19 @@ func parseConfig(m map[string]interface{}) (*config, error) { } func registerMimeTypes(mappingFile string) error { - mimeTypes := map[string]string{} - if mappingFile != "" { f, err := ioutil.ReadFile(mappingFile) if err != nil { return fmt.Errorf("storageprovider: error reading the custom mime types file: +%v", err) } - mimes := []*mimeType{} - err = json.Unmarshal(f, &mimes) + mimeTypes := map[string]string{} + err = json.Unmarshal(f, &mimeTypes) if err != nil { return fmt.Errorf("storageprovider: error unmarshalling the custom mime types file: +%v", err) } - for _, m := range mimes { - if _, found := mimeTypes[m.FileExt]; found { - return fmt.Errorf("storageprovider: mimetypes mapping error, file extension \"%s\" is mapped to multiple mime types", m.FileExt) - } - mimeTypes[m.FileExt] = m.MimeType - } - - // now register all mime types that were read - for k, v := range mimeTypes { - mime.RegisterMime(k, v) + // register all mime types that were read + for e, m := range mimeTypes { + mime.RegisterMime(e, m) } } return nil @@ -214,7 +200,7 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { } // read and register custom mime types if configured - err = registerMimeTypes(c.CustomMimeTypes) + err = registerMimeTypes(c.CustomMimeTypesJSON) if err != nil { return nil, err }