diff --git a/changelog/unreleased/mimetypes-config.md b/changelog/unreleased/mimetypes-config.md new file mode 100644 index 0000000000..680115abaa --- /dev/null +++ b/changelog/unreleased/mimetypes-config.md @@ -0,0 +1,5 @@ +Enhancement: Externalize custom mime types configuration for storage providers + +Added ability to configure custom mime types in an external JSON file, such that it can be reused when many storage providers are deployed at the same time. + +https://github.com/cs3org/reva/pull/2613 diff --git a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md index 728c382f59..0a9ff8c468 100644 --- a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="mount_path" type="string" default="/" %}} -The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L56) +The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58) {{< highlight toml >}} [grpc.services.storageprovider] mount_path = "/" @@ -17,7 +17,7 @@ mount_path = "/" {{% /dir %}} {{% dir name="mount_id" type="string" default="-" %}} -The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L57) +The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59) {{< highlight toml >}} [grpc.services.storageprovider] mount_id = "-" @@ -25,7 +25,7 @@ mount_id = "-" {{% /dir %}} {{% dir name="driver" type="string" default="localhome" %}} -The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58) +The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60) {{< highlight toml >}} [grpc.services.storageprovider] driver = "localhome" @@ -33,7 +33,7 @@ driver = "localhome" {{% /dir %}} {{% dir name="drivers" type="map[string]map[string]interface{}" default="localhome" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61) {{< highlight toml >}} [grpc.services.storageprovider.drivers.localhome] root = "/var/tmp/reva/" @@ -44,7 +44,7 @@ user_layout = "{{.Username}}" {{% /dir %}} {{% dir name="tmp_folder" type="string" default="/var/tmp" %}} -Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60) +Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62) {{< highlight toml >}} [grpc.services.storageprovider] tmp_folder = "/var/tmp" @@ -52,7 +52,7 @@ tmp_folder = "/var/tmp" {{% /dir %}} {{% dir name="data_server_url" type="string" default="http://localhost/data" %}} -The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61) +The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) {{< highlight toml >}} [grpc.services.storageprovider] data_server_url = "http://localhost/data" @@ -60,7 +60,7 @@ data_server_url = "http://localhost/data" {{% /dir %}} {{% dir name="expose_data_server" type="bool" default=false %}} -Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62) +Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) {{< highlight toml >}} [grpc.services.storageprovider] expose_data_server = false @@ -68,18 +68,18 @@ expose_data_server = false {{% /dir %}} {{% dir name="available_checksums" type="map[string]uint32" default=nil %}} -List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) +List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65) {{< highlight toml >}} [grpc.services.storageprovider] available_checksums = nil {{< /highlight >}} {{% /dir %}} -{{% dir name="mimetypes" type="map[string]string" default=nil %}} -List of supported mime types and corresponding file extensions. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) +{{% dir name="custom_mimetypes_json" type="string" default="nil" %}} +An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66) {{< highlight toml >}} [grpc.services.storageprovider] -mimetypes = nil +custom_mimetypes_json = "nil" {{< /highlight >}} {{% /dir %}} diff --git a/examples/nextcloud-integration/custom-mime-types-demo.json b/examples/nextcloud-integration/custom-mime-types-demo.json new file mode 100644 index 0000000000..a48c986790 --- /dev/null +++ b/examples/nextcloud-integration/custom-mime-types-demo.json @@ -0,0 +1,4 @@ +{ + ".zmd": "application/compressed-markdown", + ".zep": "application/compressed-etherpad" +} diff --git a/examples/nextcloud-integration/revad.toml b/examples/nextcloud-integration/revad.toml index 6c9b0c0615..52913b88ad 100644 --- a/examples/nextcloud-integration/revad.toml +++ b/examples/nextcloud-integration/revad.toml @@ -90,9 +90,7 @@ expose_data_server = true data_server_url = "http://127.0.0.1:19001/data" enable_home_creation = true disable_tus = true - -[grpc.services.storageprovider.mimetypes] -".zmd" = "application/compressed-markdown" +custom_mime_types_json = "custom-mime-types-demo.json" [grpc.services.storageprovider.drivers.nextcloud] end_point = "http://localhost/apps/sciencemesh/" diff --git a/examples/ocmd/ocmd-server-1.toml b/examples/ocmd/ocmd-server-1.toml index dad77ce087..2e41678afb 100644 --- a/examples/ocmd/ocmd-server-1.toml +++ b/examples/ocmd/ocmd-server-1.toml @@ -97,9 +97,6 @@ expose_data_server = true data_server_url = "http://localhost:19001/data" enable_home_creation = true -[grpc.services.storageprovider.mimetypes] -".zmd" = "application/compressed-markdown" - [grpc.services.storageprovider.drivers.localhome] user_layout = "{{.Username}}" diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index fd72dbcb7c..a988678110 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -20,7 +20,9 @@ package storageprovider import ( "context" + "encoding/json" "fmt" + "io/ioutil" "net/url" "os" "path" @@ -53,15 +55,15 @@ func init() { } type config struct { - MountPath string `mapstructure:"mount_path" docs:"/;The path where the file system would be mounted."` - MountID string `mapstructure:"mount_id" docs:"-;The ID of the mounted file system."` - Driver string `mapstructure:"driver" docs:"localhome;The storage driver to be used."` - Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go"` - TmpFolder string `mapstructure:"tmp_folder" docs:"/var/tmp;Path to temporary folder."` - DataServerURL string `mapstructure:"data_server_url" docs:"http://localhost/data;The URL for the data server."` - ExposeDataServer bool `mapstructure:"expose_data_server" docs:"false;Whether to expose data server."` // if true the client will be able to upload/download directly to it - AvailableXS map[string]uint32 `mapstructure:"available_checksums" docs:"nil;List of available checksums."` - MimeTypes map[string]string `mapstructure:"mimetypes" docs:"nil;List of supported mime types and corresponding file extensions."` + MountPath string `mapstructure:"mount_path" docs:"/;The path where the file system would be mounted."` + MountID string `mapstructure:"mount_id" docs:"-;The ID of the mounted file system."` + Driver string `mapstructure:"driver" docs:"localhome;The storage driver to be used."` + Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go"` + TmpFolder string `mapstructure:"tmp_folder" docs:"/var/tmp;Path to temporary folder."` + DataServerURL string `mapstructure:"data_server_url" docs:"http://localhost/data;The URL for the data server."` + ExposeDataServer bool `mapstructure:"expose_data_server" docs:"false;Whether to expose data server."` // if true the client will be able to upload/download directly to it + AvailableXS map[string]uint32 `mapstructure:"available_checksums" docs:"nil;List of available checksums."` + CustomMimeTypesJSON string `mapstructure:"custom_mimetypes_json" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."` } func (c *config) init() { @@ -94,10 +96,6 @@ func (c *config) init() { if len(c.AvailableXS) == 0 { c.AvailableXS = map[string]uint32{"md5": 100, "unset": 1000} } - if c.MimeTypes == nil || len(c.MimeTypes) == 0 { - c.MimeTypes = map[string]string{".zmd": "application/compressed-markdown"} - } - } type service struct { @@ -144,6 +142,25 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } +func registerMimeTypes(mappingFile string) error { + if mappingFile != "" { + f, err := ioutil.ReadFile(mappingFile) + if err != nil { + return fmt.Errorf("storageprovider: error reading the custom mime types file: +%v", err) + } + mimeTypes := map[string]string{} + err = json.Unmarshal(f, &mimeTypes) + if err != nil { + return fmt.Errorf("storageprovider: error unmarshalling the custom mime types file: +%v", err) + } + // register all mime types that were read + for e, m := range mimeTypes { + mime.RegisterMime(e, m) + } + } + return nil +} + // New creates a new storage provider svc func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { @@ -182,7 +199,11 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return nil, errtypes.NotFound("no available checksum, please set in config") } - registerMimeTypes(c.MimeTypes) + // read and register custom mime types if configured + err = registerMimeTypes(c.CustomMimeTypesJSON) + if err != nil { + return nil, err + } service := &service{ conf: c, @@ -197,12 +218,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return service, nil } -func registerMimeTypes(mimes map[string]string) { - for k, v := range mimes { - mime.RegisterMime(k, v) - } -} - func (s *service) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitraryMetadataRequest) (*provider.SetArbitraryMetadataResponse, error) { newRef, err := s.unwrap(ctx, req.Ref) if err != nil {