From adb1c6952c98926b742e8417f62e794297460349 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Mon, 28 Sep 2020 21:15:44 -0300 Subject: [PATCH 01/15] initial impl. s3fs mount --- package.json | 2 +- s3fs-volume-plugin/Dockerfile | 15 ++++++ s3fs-volume-plugin/README.md | 82 +++++++++++++++++++++++++++++++++ s3fs-volume-plugin/config.json | 52 +++++++++++++++++++++ s3fs-volume-plugin/main.go | 69 +++++++++++++++++++++++++++ s3fs-volume-plugin/main_test.go | 37 +++++++++++++++ 6 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 s3fs-volume-plugin/Dockerfile create mode 100644 s3fs-volume-plugin/README.md create mode 100644 s3fs-volume-plugin/config.json create mode 100644 s3fs-volume-plugin/main.go create mode 100644 s3fs-volume-plugin/main_test.go diff --git a/package.json b/package.json index 55cba09..baa1907 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "docker-volume-plugins", - "version": "2.0.3", + "version": "2.0.4", "description": "Docker Managed Volume Plugins\r =============================", "main": "index.js", "scripts": { diff --git a/s3fs-volume-plugin/Dockerfile b/s3fs-volume-plugin/Dockerfile new file mode 100644 index 0000000..f7fb34a --- /dev/null +++ b/s3fs-volume-plugin/Dockerfile @@ -0,0 +1,15 @@ +FROM oraclelinux:7-slim +ENV TINI_VERSION v0.18.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +RUN yum install -q -y oracle-epel-release-el7 +RUN yum install -q -y git fuse s3fs-fuse attr && \ + curl --silent -L https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz | tar -C /usr/local -zxf - +RUN /usr/local/go/bin/go get github.com/trajano/docker-volume-plugins/s3fs-volume-plugin && \ + mv $HOME/go/bin/s3fs-volume-plugin / && \ + rm -rf $HOME/go /usr/local/go && \ + yum remove -q -y git && \ + yum autoremove -q -y && \ + yum clean all && \ + rm -rf /var/cache/yum /var/log/anaconda /var/cache/yum /etc/mtab && \ + rm /var/log/lastlog /var/log/tallylog diff --git a/s3fs-volume-plugin/README.md b/s3fs-volume-plugin/README.md new file mode 100644 index 0000000..ebde2c3 --- /dev/null +++ b/s3fs-volume-plugin/README.md @@ -0,0 +1,82 @@ +S3Fs Volume Plugin +======================= + +This is a managed Docker volume plugin to allow Docker containers to access S3Fs volumes. The S3Fs client does not need to be installed on the host and everything is managed within the plugin. + +### Caveats: + +- Requires Docker 18.03-1 at minimum. +- This is a managed plugin only, no legacy support. +- In order to properly support versions use `--alias` when installing the plugin. +- This only supports one S3Fs cluster per instance use `--alias` to define separate instances +- The value of `SERVERS` is initially blank it needs `docker plugin s3fs set AWSACCESSKEYID=key;docker plugin s3fs set AWSSECRETACCESSKEY=secret` if it is set then it will be used for all servers and low level options will not be allowed. Primarily this is to control what the deployed stacks can perform. +- **There is no robust error handling. So garbage in -> garbage out** + +## Operating modes + +There are three operating modes listed in order of preference. Each are mutually exclusive and will result in an error when performing a `docker volume create` if more than one operating mode is configured. + +### Just the name + +This is the *recommended* approach for production systems as it will prevent stacks from specifying any random server. It also prevents the stack configuration file from containing environment specific servers and instead defers that knowledge to the plugin only which is set on the node level. This relies on `SERVERS` being configured and will use the name as the volume mount set by [`docker plugin set`](https://docs.docker.com/engine/reference/commandline/plugin_set/). This can be done in an automated fashion as: + + docker plugin install --alias PLUGINALIAS \ + trajano/s3fs-volume-plugin \ + --grant-all-permissions --disable + docker plugin set AWSACCESSKEYID=key + docker plugin set AWSSECRETACCESSKEY=secret + docker plugin enable PLUGINALIAS + +If there is a need to have a different set of servers, a separate plugin alias should be created with a different set of servers. + +Example in docker-compose.yml: + + volumes: + sample: + driver: s3fs + name: "bucket/subdir" + +The `volumes.x.name` specifies the volume and optionally a subdirectory mount. The value of `name` will be used as the `--volfile-id` and `--subdir-mount`. Note that `volumes.x.name` must not start with `/`. + +### Specify the s3fs driver opts + +This uses the `driver_opts.servers` to define a comma separated list of servers. The rules for specifying the volume is the same as the previous section. + +Example in docker-compose.yml assuming the alias was set as `S3Fs`: + + volumes: + sample: + driver: s3fs + driver_opts: + s3fsopts: nomultipart,use_path_request_style + name: "bucket/subdir" + +The `volumes.x.name` specifies the volume and optionally a subdirectory mount. The value of `name` will be used as the `--volfile-id` and `--subdir-mount`. Note that `volumes.x.name` must not start with `/`. The values above correspond to the following mounting command: + + s3fs -o nomultipart -o use_path_request_style -o bucket=bucket \ + -o servicepath=subdir [generated_mount_point] + +### Specify the options + +This passes the `driver_opts.glusteropts` to the `S3Fs` command followed by the generated mount point. This is the most flexible method and gives full range to the options of the S3Fs FUSE client. Example in docker-compose.yml assuming the alias was set as `S3Fs`: + + volumes: + sample: + driver: s3fs + driver_opts: + s3fsopts: "nomultipart,use_path_request_style,bucket=bucket,servicepath=subdir" + name: "whatever" + +The value of `name` will not be used for mounting; the value of `driver_opts.S3Fsopts` is expected to have all the volume connection information. + +## Testing outside the swarm + +This is an example of mounting and testing a store outside the swarm. It is assuming the server is called `store1` and the volume name is `trajano`. + + docker plugin install trajano/s3fs-volume-plugin --grant-all-permissions --disable + docker plugin set AWSACCESSKEYID=key + docker plugin set AWSSECRETACCESSKEY=secret + docker plugin set DEFAULT_S3FSOPTS="nomultipart,use_path_request_style" + docker plugin enable trajano/s3fs-volume-plugin + docker volume create -d trajano/s3fs-volume-plugin --opt bucket=mybucket trajano + docker run -it -v trajano:/mnt alpine diff --git a/s3fs-volume-plugin/config.json b/s3fs-volume-plugin/config.json new file mode 100644 index 0000000..16607c4 --- /dev/null +++ b/s3fs-volume-plugin/config.json @@ -0,0 +1,52 @@ +{ + "description": "S3FS plugin for Docker", + "documentation": "https://github.com/trajano/docker-volume-plugins/", + "entrypoint": [ + "/tini", + "--", + "/glusterfs-volume-plugin" + ], + "env": [ + { + "name": "AWSACCESSKEYID", + "settable": [ + "value" + ], + "value": "" + }, + { + "name": "AWSSECRETACCESSKEY", + "settable": [ + "value" + ], + "value": "" + }, + { + "name": "DEFAULT_S3FSOPTS", + "settable": [ + "value" + ], + "value": "" + } + ], + "network": { + "type": "host" + }, + "propagatedMount": "/var/lib/docker-volumes", + "interface": { + "types": [ + "docker.volumedriver/1.0" + ], + "socket": "s3fs.sock" + }, + "linux": { + "capabilities": [ + "CAP_SYS_ADMIN" + ], + "devices": [ + { + "path": "/dev/fuse" + } + ] + } +} \ No newline at end of file diff --git a/s3fs-volume-plugin/main.go b/s3fs-volume-plugin/main.go new file mode 100644 index 0000000..dc43d9d --- /dev/null +++ b/s3fs-volume-plugin/main.go @@ -0,0 +1,69 @@ +package main + +import ( + "log" + "os" + "strings" + + "github.com/docker/go-plugins-helpers/volume" + mountedvolume "github.com/microsoft/vscode-remote-try-go/docker-volume-plugins/mounted-volume" +) + +type s3fsDriver struct { + defaultS3fsopts string + mountedvolume.Driver +} + +func (p *s3fsDriver) Validate(req *volume.CreateRequest) error { + + return nil +} + +func (p *s3fsDriver) MountOptions(req *volume.CreateRequest) []string { + + s3fsopts, s3fsoptsInOpts := req.Options["s3fsopts"] + + var s3fsoptsArray []string + if s3fsoptsInOpts { + s3fsoptsArray = append(s3fsoptsArray, strings.Split(s3fsopts, ",")...) + } else { + s3fsoptsArray = append(s3fsoptsArray, strings.Split(p.defaultS3fsopts, ",")...) + } + s3fsoptsArray = AppendBucketOptionsByVolumeName(s3fsoptsArray, req.Name) + + return []string{"-o", strings.Join(s3fsoptsArray, ",")} +} + +func (p *s3fsDriver) PreMount(req *volume.MountRequest) error { + return nil +} + +func (p *s3fsDriver) PostMount(req *volume.MountRequest) { +} + +// AppendBucketOptionsByVolumeName appends the command line arguments into the current argument list given the volume name +func AppendBucketOptionsByVolumeName(args []string, volumeName string) []string { + parts := strings.SplitN(volumeName, "/", 2) + ret := append(args, "bucket="+parts[0]) + if len(parts) == 2 { + ret = append(ret, "servicepath="+parts[1]) + } + return ret +} + +func buildDriver() *s3fsDriver { + defaultS3fsopts := os.Getenv("DEFAULT_S3FSOPTS") + d := &s3fsDriver{ + Driver: *mountedvolume.NewDriver("s3fs", false, "s3fs", "local"), + defaultS3fsopts: defaultS3fsopts, + } + d.Init(d) + return d +} + +func main() { + log.SetFlags(0) + d := buildDriver() + defer d.Close() + d.ServeUnix() +} diff --git a/s3fs-volume-plugin/main_test.go b/s3fs-volume-plugin/main_test.go new file mode 100644 index 0000000..b05ff4a --- /dev/null +++ b/s3fs-volume-plugin/main_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "reflect" + "testing" +) + +func TestVolumeCalculation(t *testing.T) { + + var calculated = AppendBucketOptionsByVolumeName([]string{"mount"}, "simplevolume") + var expected = []string{"mount", "-o bucket=simplevolume"} + if !reflect.DeepEqual(calculated, expected) { + fmt.Errorf("%v didn't match expected", calculated) + t.Fail() + } +} + +func TestVolumeCalculationOneLevel(t *testing.T) { + + var calculated = AppendBucketOptionsByVolumeName([]string{"mount"}, "simplevolume/levelone") + var expected = []string{"mount", "-o bucket=simplevolume", "-o servicepath=/levelone"} + if !reflect.DeepEqual(calculated, expected) { + fmt.Errorf("%v didn't match expected", calculated) + t.Fail() + } +} + +func TestVolumeCalculationTwoLevels(t *testing.T) { + + var calculated = AppendBucketOptionsByVolumeName([]string{"mount"}, "simplevolume/levelone/level2") + var expected = []string{"mount", "-o bucket=simplevolume", "-o servicepath=/levelone/level2"} + if !reflect.DeepEqual(calculated, expected) { + fmt.Errorf("%v didn't match expected", calculated) + t.Fail() + } +} From 34c9b6761250a42d019af9f276bd0614e7224575 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 00:47:46 +0000 Subject: [PATCH 02/15] renamed default var --- s3fs-volume-plugin/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s3fs-volume-plugin/main.go b/s3fs-volume-plugin/main.go index dc43d9d..8097e28 100644 --- a/s3fs-volume-plugin/main.go +++ b/s3fs-volume-plugin/main.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/docker/go-plugins-helpers/volume" - mountedvolume "github.com/microsoft/vscode-remote-try-go/docker-volume-plugins/mounted-volume" + "github.com/trajano/docker-volume-plugins/mounted-volume" ) type s3fsDriver struct { @@ -52,10 +52,10 @@ func AppendBucketOptionsByVolumeName(args []string, volumeName string) []string } func buildDriver() *s3fsDriver { - defaultS3fsopts := os.Getenv("DEFAULT_S3FSOPTS") + defaultopts := os.Getenv("DEFAULT_S3FSOPTS") d := &s3fsDriver{ Driver: *mountedvolume.NewDriver("s3fs", false, "s3fs", "local"), - defaultS3fsopts: defaultS3fsopts, + defaultS3fsopts: defaultopts, } d.Init(d) return d From 3fddc3e8b2bc3d20d3fc423993a14c334d3dd5bb Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Mon, 28 Sep 2020 22:09:12 -0300 Subject: [PATCH 03/15] renamed default var --- s3fs-volume-plugin/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s3fs-volume-plugin/main.go b/s3fs-volume-plugin/main.go index dc43d9d..3772444 100644 --- a/s3fs-volume-plugin/main.go +++ b/s3fs-volume-plugin/main.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/docker/go-plugins-helpers/volume" - mountedvolume "github.com/microsoft/vscode-remote-try-go/docker-volume-plugins/mounted-volume" + "github.com/microsoft/vscode-remote-try-go/docker-volume-plugins/mounted-volume" ) type s3fsDriver struct { @@ -52,10 +52,10 @@ func AppendBucketOptionsByVolumeName(args []string, volumeName string) []string } func buildDriver() *s3fsDriver { - defaultS3fsopts := os.Getenv("DEFAULT_S3FSOPTS") + defaultsopts := os.Getenv("DEFAULT_S3FSOPTS") d := &s3fsDriver{ Driver: *mountedvolume.NewDriver("s3fs", false, "s3fs", "local"), - defaultS3fsopts: defaultS3fsopts, + defaultS3fsopts: defaultsopts, } d.Init(d) return d From ac02071938f34327082c3422c36ddaa58195c468 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Mon, 28 Sep 2020 22:12:00 -0300 Subject: [PATCH 04/15] fix import pkg --- s3fs-volume-plugin/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3fs-volume-plugin/main.go b/s3fs-volume-plugin/main.go index 3772444..ce52b5e 100644 --- a/s3fs-volume-plugin/main.go +++ b/s3fs-volume-plugin/main.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/docker/go-plugins-helpers/volume" - "github.com/microsoft/vscode-remote-try-go/docker-volume-plugins/mounted-volume" + "github.com/trajano/docker-volume-plugins/mounted-volume" ) type s3fsDriver struct { From b2a4d1ccb52c58049d9e03172ed1f2c6ec9ae185 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 11:39:18 -0300 Subject: [PATCH 05/15] auto-remove is not working well --- s3fs-volume-plugin/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/s3fs-volume-plugin/Dockerfile b/s3fs-volume-plugin/Dockerfile index f7fb34a..2871427 100644 --- a/s3fs-volume-plugin/Dockerfile +++ b/s3fs-volume-plugin/Dockerfile @@ -9,7 +9,6 @@ RUN /usr/local/go/bin/go get github.com/trajano/docker-volume-plugins/s3fs-volum mv $HOME/go/bin/s3fs-volume-plugin / && \ rm -rf $HOME/go /usr/local/go && \ yum remove -q -y git && \ - yum autoremove -q -y && \ yum clean all && \ rm -rf /var/cache/yum /var/log/anaconda /var/cache/yum /etc/mtab && \ rm /var/log/lastlog /var/log/tallylog From 816b1a04cbc8dc304b8644cdeac6cd4372c5eb1f Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 14:53:40 +0000 Subject: [PATCH 06/15] update readme --- s3fs-volume-plugin/README.md | 37 +++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/s3fs-volume-plugin/README.md b/s3fs-volume-plugin/README.md index ebde2c3..ef3528c 100644 --- a/s3fs-volume-plugin/README.md +++ b/s3fs-volume-plugin/README.md @@ -9,7 +9,7 @@ This is a managed Docker volume plugin to allow Docker containers to access S3Fs - This is a managed plugin only, no legacy support. - In order to properly support versions use `--alias` when installing the plugin. - This only supports one S3Fs cluster per instance use `--alias` to define separate instances -- The value of `SERVERS` is initially blank it needs `docker plugin s3fs set AWSACCESSKEYID=key;docker plugin s3fs set AWSSECRETACCESSKEY=secret` if it is set then it will be used for all servers and low level options will not be allowed. Primarily this is to control what the deployed stacks can perform. +- The value of `AWSACCESSKEYID/AWSSECRETACCESSKEY` is initially blank it needs `docker plugin s3fs set AWSACCESSKEYID=key;docker plugin s3fs set AWSSECRETACCESSKEY=secret` if it is set then it will be used for all servers and low level options will not be allowed. Primarily this is to control what the deployed stacks can perform. - **There is no robust error handling. So garbage in -> garbage out** ## Operating modes @@ -23,8 +23,8 @@ This is the *recommended* approach for production systems as it will prevent sta docker plugin install --alias PLUGINALIAS \ trajano/s3fs-volume-plugin \ --grant-all-permissions --disable - docker plugin set AWSACCESSKEYID=key - docker plugin set AWSSECRETACCESSKEY=secret + docker plugin set PLUGINALIAS AWSACCESSKEYID=key + docker plugin set PLUGINALIAS AWSSECRETACCESSKEY=secret docker plugin enable PLUGINALIAS If there is a need to have a different set of servers, a separate plugin alias should be created with a different set of servers. @@ -36,13 +36,13 @@ Example in docker-compose.yml: driver: s3fs name: "bucket/subdir" -The `volumes.x.name` specifies the volume and optionally a subdirectory mount. The value of `name` will be used as the `--volfile-id` and `--subdir-mount`. Note that `volumes.x.name` must not start with `/`. +The `volumes.x.name` specifies the bucket and optionally a subdirectory mount. The value of `name` will be used as the `-o bucket=` and `-o servicepath=` in s3fs fuse mount. Note that `volumes.x.name` must not start with `/`. ### Specify the s3fs driver opts -This uses the `driver_opts.servers` to define a comma separated list of servers. The rules for specifying the volume is the same as the previous section. +This uses the `driver_opts.servers` to define a comma separated list s3fs options. The rules for specifying the volume is the same as the previous section. -Example in docker-compose.yml assuming the alias was set as `S3Fs`: +Example in docker-compose.yml assuming the alias was set as `s3fs`: volumes: sample: @@ -51,14 +51,13 @@ Example in docker-compose.yml assuming the alias was set as `S3Fs`: s3fsopts: nomultipart,use_path_request_style name: "bucket/subdir" -The `volumes.x.name` specifies the volume and optionally a subdirectory mount. The value of `name` will be used as the `--volfile-id` and `--subdir-mount`. Note that `volumes.x.name` must not start with `/`. The values above correspond to the following mounting command: +The `volumes.x.name` specifies the bucket and optionally a subdirectory mount. The value of `name` will be used as the `-o bucket=` and `-o servicepath=`. Note that `volumes.x.name` must not start with `/`. The values above correspond to the following mounting command: - s3fs -o nomultipart -o use_path_request_style -o bucket=bucket \ - -o servicepath=subdir [generated_mount_point] + s3fs -o nomultipart,use_path_request_style,bucket=bucket,servicepath=subdir [generated_mount_point] ### Specify the options -This passes the `driver_opts.glusteropts` to the `S3Fs` command followed by the generated mount point. This is the most flexible method and gives full range to the options of the S3Fs FUSE client. Example in docker-compose.yml assuming the alias was set as `S3Fs`: +This passes the `driver_opts.glusteropts` to the `s3fs` command followed by the generated mount point. This is the most flexible method and gives full range to the options of the S3Fs FUSE client. Example in docker-compose.yml assuming the alias was set as `s3fs`: volumes: sample: @@ -67,7 +66,7 @@ This passes the `driver_opts.glusteropts` to the `S3Fs` command followed by the s3fsopts: "nomultipart,use_path_request_style,bucket=bucket,servicepath=subdir" name: "whatever" -The value of `name` will not be used for mounting; the value of `driver_opts.S3Fsopts` is expected to have all the volume connection information. +The value of `name` will not be used for mounting; the value of `driver_opts.s3fsopts` is expected to have all the volume connection information. ## Testing outside the swarm @@ -78,5 +77,17 @@ This is an example of mounting and testing a store outside the swarm. It is ass docker plugin set AWSSECRETACCESSKEY=secret docker plugin set DEFAULT_S3FSOPTS="nomultipart,use_path_request_style" docker plugin enable trajano/s3fs-volume-plugin - docker volume create -d trajano/s3fs-volume-plugin --opt bucket=mybucket trajano - docker run -it -v trajano:/mnt alpine + docker volume create -d trajano/s3fs-volume-plugin mybucket + docker run -it -v mybucket:/mnt alpine + +## Testing with Oracle Cloud Object storage + +Sample usage Oracle Object Storage in S3 compatibilty mode, replace tenant_id and region_id with a proper value: + + docker plugin install trajano/s3fs-volume-plugin --grant-all-permissions --disable + docker plugin set AWSACCESSKEYID=key + docker plugin set AWSSECRETACCESSKEY=secret + docker plugin set DEFAULT_S3FSOPTS="nomultipart,use_path_request_style,url=https://[tenant_id].compat.objectstorage.[region-id].oraclecloud.com/" + docker plugin enable trajano/s3fs-volume-plugin + docker volume create -d trajano/s3fs-volume-plugin mybucket + docker run -it -v mybucket:/mnt alpine \ No newline at end of file From 580d8905d8ef1444a8c548c36cc5746f56f10c76 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 11:55:25 -0300 Subject: [PATCH 07/15] Update README.md --- s3fs-volume-plugin/README.md | 37 +++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/s3fs-volume-plugin/README.md b/s3fs-volume-plugin/README.md index ebde2c3..4b19990 100644 --- a/s3fs-volume-plugin/README.md +++ b/s3fs-volume-plugin/README.md @@ -9,7 +9,7 @@ This is a managed Docker volume plugin to allow Docker containers to access S3Fs - This is a managed plugin only, no legacy support. - In order to properly support versions use `--alias` when installing the plugin. - This only supports one S3Fs cluster per instance use `--alias` to define separate instances -- The value of `SERVERS` is initially blank it needs `docker plugin s3fs set AWSACCESSKEYID=key;docker plugin s3fs set AWSSECRETACCESSKEY=secret` if it is set then it will be used for all servers and low level options will not be allowed. Primarily this is to control what the deployed stacks can perform. +- The value of `AWSACCESSKEYID/AWSSECRETACCESSKEY` is initially blank it needs `docker plugin s3fs set AWSACCESSKEYID=key;docker plugin s3fs set AWSSECRETACCESSKEY=secret` if it is set then it will be used for all servers and low level options will not be allowed. Primarily this is to control what the deployed stacks can perform. - **There is no robust error handling. So garbage in -> garbage out** ## Operating modes @@ -23,8 +23,8 @@ This is the *recommended* approach for production systems as it will prevent sta docker plugin install --alias PLUGINALIAS \ trajano/s3fs-volume-plugin \ --grant-all-permissions --disable - docker plugin set AWSACCESSKEYID=key - docker plugin set AWSSECRETACCESSKEY=secret + docker plugin set PLUGINALIAS AWSACCESSKEYID=key + docker plugin set PLUGINALIAS AWSSECRETACCESSKEY=secret docker plugin enable PLUGINALIAS If there is a need to have a different set of servers, a separate plugin alias should be created with a different set of servers. @@ -36,13 +36,13 @@ Example in docker-compose.yml: driver: s3fs name: "bucket/subdir" -The `volumes.x.name` specifies the volume and optionally a subdirectory mount. The value of `name` will be used as the `--volfile-id` and `--subdir-mount`. Note that `volumes.x.name` must not start with `/`. +The `volumes.x.name` specifies the bucket and optionally a subdirectory mount. The value of `name` will be used as the `-o bucket=` and `-o servicepath=` in s3fs fuse mount. Note that `volumes.x.name` must not start with `/`. ### Specify the s3fs driver opts -This uses the `driver_opts.servers` to define a comma separated list of servers. The rules for specifying the volume is the same as the previous section. +This uses the `driver_opts.servers` to define a comma separated list s3fs options. The rules for specifying the volume is the same as the previous section. -Example in docker-compose.yml assuming the alias was set as `S3Fs`: +Example in docker-compose.yml assuming the alias was set as `s3fs`: volumes: sample: @@ -51,14 +51,13 @@ Example in docker-compose.yml assuming the alias was set as `S3Fs`: s3fsopts: nomultipart,use_path_request_style name: "bucket/subdir" -The `volumes.x.name` specifies the volume and optionally a subdirectory mount. The value of `name` will be used as the `--volfile-id` and `--subdir-mount`. Note that `volumes.x.name` must not start with `/`. The values above correspond to the following mounting command: +The `volumes.x.name` specifies the bucket and optionally a subdirectory mount. The value of `name` will be used as the `-o bucket=` and `-o servicepath=`. Note that `volumes.x.name` must not start with `/`. The values above correspond to the following mounting command: - s3fs -o nomultipart -o use_path_request_style -o bucket=bucket \ - -o servicepath=subdir [generated_mount_point] + s3fs -o nomultipart,use_path_request_style,bucket=bucket,servicepath=subdir [generated_mount_point] ### Specify the options -This passes the `driver_opts.glusteropts` to the `S3Fs` command followed by the generated mount point. This is the most flexible method and gives full range to the options of the S3Fs FUSE client. Example in docker-compose.yml assuming the alias was set as `S3Fs`: +This passes the `driver_opts.glusteropts` to the `s3fs` command followed by the generated mount point. This is the most flexible method and gives full range to the options of the S3Fs FUSE client. Example in docker-compose.yml assuming the alias was set as `s3fs`: volumes: sample: @@ -67,7 +66,7 @@ This passes the `driver_opts.glusteropts` to the `S3Fs` command followed by the s3fsopts: "nomultipart,use_path_request_style,bucket=bucket,servicepath=subdir" name: "whatever" -The value of `name` will not be used for mounting; the value of `driver_opts.S3Fsopts` is expected to have all the volume connection information. +The value of `name` will not be used for mounting; the value of `driver_opts.s3fsopts` is expected to have all the volume connection information. ## Testing outside the swarm @@ -78,5 +77,17 @@ This is an example of mounting and testing a store outside the swarm. It is ass docker plugin set AWSSECRETACCESSKEY=secret docker plugin set DEFAULT_S3FSOPTS="nomultipart,use_path_request_style" docker plugin enable trajano/s3fs-volume-plugin - docker volume create -d trajano/s3fs-volume-plugin --opt bucket=mybucket trajano - docker run -it -v trajano:/mnt alpine + docker volume create -d trajano/s3fs-volume-plugin mybucket + docker run -it -v mybucket:/mnt alpine + +## Testing with Oracle Cloud Object storage + +Sample usage Oracle Object Storage in S3 compatibilty mode, replace tenant_id and region_id with a proper value: + + docker plugin install trajano/s3fs-volume-plugin --grant-all-permissions --disable + docker plugin set AWSACCESSKEYID=key + docker plugin set AWSSECRETACCESSKEY=secret + docker plugin set DEFAULT_S3FSOPTS="nomultipart,use_path_request_style,url=https://[tenant_id].compat.objectstorage.[region-id].oraclecloud.com/" + docker plugin enable trajano/s3fs-volume-plugin + docker volume create -d trajano/s3fs-volume-plugin mybucket + docker run -it -v mybucket:/mnt alpine From 9062b50ab7ff22cf561700294a63a3961c238c5c Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 11:57:33 -0300 Subject: [PATCH 08/15] included FIX #29 --- glusterfs-volume-plugin/config.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/glusterfs-volume-plugin/config.json b/glusterfs-volume-plugin/config.json index 4c96a21..bddb3f6 100644 --- a/glusterfs-volume-plugin/config.json +++ b/glusterfs-volume-plugin/config.json @@ -2,6 +2,8 @@ "description": "GlusterFS plugin for Docker", "documentation": "https://github.com/trajano/docker-volume-plugins/", "entrypoint": [ + "/tini", + "--", "/glusterfs-volume-plugin" ], "env": [ @@ -33,4 +35,4 @@ } ] } -} \ No newline at end of file +} From fc782807b6be1f71e938e107c3212cd1095f6772 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 15:21:00 +0000 Subject: [PATCH 09/15] added / in servicepath --- glusterfs-volume-plugin/Dockerfile | 1 - s3fs-volume-plugin/main.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/glusterfs-volume-plugin/Dockerfile b/glusterfs-volume-plugin/Dockerfile index 1e3734d..698fb81 100644 --- a/glusterfs-volume-plugin/Dockerfile +++ b/glusterfs-volume-plugin/Dockerfile @@ -6,7 +6,6 @@ RUN /usr/local/go/bin/go get github.com/trajano/docker-volume-plugins/glusterfs- mv $HOME/go/bin/glusterfs-volume-plugin / && \ rm -rf $HOME/go /usr/local/go && \ yum remove -q -y git && \ - yum autoremove -q -y && \ yum clean all && \ rm -rf /var/cache/yum /var/log/anaconda /var/cache/yum /etc/mtab && \ rm /var/log/lastlog /var/log/tallylog diff --git a/s3fs-volume-plugin/main.go b/s3fs-volume-plugin/main.go index 8259360..a54bec0 100644 --- a/s3fs-volume-plugin/main.go +++ b/s3fs-volume-plugin/main.go @@ -46,7 +46,7 @@ func AppendBucketOptionsByVolumeName(args []string, volumeName string) []string parts := strings.SplitN(volumeName, "/", 2) ret := append(args, "bucket="+parts[0]) if len(parts) == 2 { - ret = append(ret, "servicepath="+parts[1]) + ret = append(ret, "servicepath=/"+parts[1]) } return ret } From 61a31711a53bbe57d4172093c22b4cbcd5645631 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 15:23:06 +0000 Subject: [PATCH 10/15] added tini --- glusterfs-volume-plugin/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/glusterfs-volume-plugin/Dockerfile b/glusterfs-volume-plugin/Dockerfile index 698fb81..a96f348 100644 --- a/glusterfs-volume-plugin/Dockerfile +++ b/glusterfs-volume-plugin/Dockerfile @@ -1,4 +1,7 @@ FROM oraclelinux:7-slim +ENV TINI_VERSION v0.18.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini RUN yum install -q -y oracle-gluster-release-el7 && \ yum install -q -y git glusterfs glusterfs-fuse attr && \ curl --silent -L https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz | tar -C /usr/local -zxf - From 4b5947730e0f6f6b54cd22364ec2ea54eff82f94 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 15:33:02 +0000 Subject: [PATCH 11/15] update s3fs entrypoint --- s3fs-volume-plugin/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3fs-volume-plugin/config.json b/s3fs-volume-plugin/config.json index 16607c4..dc40a42 100644 --- a/s3fs-volume-plugin/config.json +++ b/s3fs-volume-plugin/config.json @@ -4,7 +4,7 @@ "entrypoint": [ "/tini", "--", - "/glusterfs-volume-plugin" + "/s3fs-volume-plugin" ], "env": [ { From 84139943317a619a3e9d2cb52691f9ca53f6e3fb Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 17:21:18 +0000 Subject: [PATCH 12/15] fixed test suite --- s3fs-volume-plugin/main_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/s3fs-volume-plugin/main_test.go b/s3fs-volume-plugin/main_test.go index b05ff4a..4e97e87 100644 --- a/s3fs-volume-plugin/main_test.go +++ b/s3fs-volume-plugin/main_test.go @@ -8,8 +8,8 @@ import ( func TestVolumeCalculation(t *testing.T) { - var calculated = AppendBucketOptionsByVolumeName([]string{"mount"}, "simplevolume") - var expected = []string{"mount", "-o bucket=simplevolume"} + var calculated = AppendBucketOptionsByVolumeName([]string{"mount"}, "mybucket") + var expected = []string{"mount", "bucket=mybucket"} if !reflect.DeepEqual(calculated, expected) { fmt.Errorf("%v didn't match expected", calculated) t.Fail() @@ -18,8 +18,8 @@ func TestVolumeCalculation(t *testing.T) { func TestVolumeCalculationOneLevel(t *testing.T) { - var calculated = AppendBucketOptionsByVolumeName([]string{"mount"}, "simplevolume/levelone") - var expected = []string{"mount", "-o bucket=simplevolume", "-o servicepath=/levelone"} + var calculated = AppendBucketOptionsByVolumeName([]string{"mount"}, "mybucket/levelone") + var expected = []string{"mount", "bucket=mybucket", "servicepath=/levelone"} if !reflect.DeepEqual(calculated, expected) { fmt.Errorf("%v didn't match expected", calculated) t.Fail() @@ -28,8 +28,8 @@ func TestVolumeCalculationOneLevel(t *testing.T) { func TestVolumeCalculationTwoLevels(t *testing.T) { - var calculated = AppendBucketOptionsByVolumeName([]string{"mount"}, "simplevolume/levelone/level2") - var expected = []string{"mount", "-o bucket=simplevolume", "-o servicepath=/levelone/level2"} + var calculated = AppendBucketOptionsByVolumeName([]string{"mount"}, "mybucket/levelone/level2") + var expected = []string{"mount", "bucket=mybucket", "servicepath=/levelone/level2"} if !reflect.DeepEqual(calculated, expected) { fmt.Errorf("%v didn't match expected", calculated) t.Fail() From c4408962ce41ce6b5b9d79a5aed995112ae4d135 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 17:58:33 +0000 Subject: [PATCH 13/15] removed mountedvolume from import --- s3fs-volume-plugin/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3fs-volume-plugin/main.go b/s3fs-volume-plugin/main.go index a54bec0..daad419 100644 --- a/s3fs-volume-plugin/main.go +++ b/s3fs-volume-plugin/main.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/docker/go-plugins-helpers/volume" - mountedvolume "github.com/trajano/docker-volume-plugins/mounted-volume" + "github.com/trajano/docker-volume-plugins/mounted-volume" ) type s3fsDriver struct { From a7d6b0b1e9a29f31e6c361374e39b453375c7603 Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 22:43:50 +0000 Subject: [PATCH 14/15] update to go 1.15.2, sample swarm stack and enable build module --- build.sh | 1 + glusterfs-volume-plugin/Dockerfile | 2 +- s3fs-volume-plugin/Dockerfile | 2 +- s3fs-volume-plugin/docker-compose.yml | 16 ++++++++++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 s3fs-volume-plugin/docker-compose.yml diff --git a/build.sh b/build.sh index 6eafb32..326c34e 100755 --- a/build.sh +++ b/build.sh @@ -28,6 +28,7 @@ build() { fi } build glusterfs-volume-plugin +build s3fs-volume-plugin build cifs-volume-plugin build nfs-volume-plugin build centos-mounted-volume-plugin diff --git a/glusterfs-volume-plugin/Dockerfile b/glusterfs-volume-plugin/Dockerfile index a96f348..3fab445 100644 --- a/glusterfs-volume-plugin/Dockerfile +++ b/glusterfs-volume-plugin/Dockerfile @@ -4,7 +4,7 @@ ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini RUN chmod +x /tini RUN yum install -q -y oracle-gluster-release-el7 && \ yum install -q -y git glusterfs glusterfs-fuse attr && \ - curl --silent -L https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz | tar -C /usr/local -zxf - + curl --silent -L https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz | tar -C /usr/local -zxf - RUN /usr/local/go/bin/go get github.com/trajano/docker-volume-plugins/glusterfs-volume-plugin && \ mv $HOME/go/bin/glusterfs-volume-plugin / && \ rm -rf $HOME/go /usr/local/go && \ diff --git a/s3fs-volume-plugin/Dockerfile b/s3fs-volume-plugin/Dockerfile index 2871427..2c8aa37 100644 --- a/s3fs-volume-plugin/Dockerfile +++ b/s3fs-volume-plugin/Dockerfile @@ -4,7 +4,7 @@ ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini RUN chmod +x /tini RUN yum install -q -y oracle-epel-release-el7 RUN yum install -q -y git fuse s3fs-fuse attr && \ - curl --silent -L https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz | tar -C /usr/local -zxf - + curl --silent -L https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz | tar -C /usr/local -zxf - RUN /usr/local/go/bin/go get github.com/trajano/docker-volume-plugins/s3fs-volume-plugin && \ mv $HOME/go/bin/s3fs-volume-plugin / && \ rm -rf $HOME/go /usr/local/go && \ diff --git a/s3fs-volume-plugin/docker-compose.yml b/s3fs-volume-plugin/docker-compose.yml new file mode 100644 index 0000000..a436fa8 --- /dev/null +++ b/s3fs-volume-plugin/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3.7' + +volumes: + # docker plugin set trajano/s3fs-volume-plugin:latest AWSACCESSKEYID=key + # docker plugin set trajano/s3fs-volume-plugin:latest AWSSECRETACCESSKEY="secret" + # docker plugin set trajano/s3fs-volume-plugin:latest DEFAULT_S3FSOPTS="nomultipart,use_path_request_style,url=https://[tenant].compat.objectstorage.[region].oraclecloud.com/" + mybucket: + driver: trajano/s3fs-volume-plugin:latest + name: "docker-shared-bucket" + +services: + myapp: + image: alpine + command: sleep 1d + volumes: + - mybucket:/mnt From e8b04a0895e279c20ec0921c4092bb59837d48fd Mon Sep 17 00:00:00 2001 From: Marcelo Ochoa Date: Tue, 29 Sep 2020 22:56:02 +0000 Subject: [PATCH 15/15] fixed some comments --- s3fs-volume-plugin/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/s3fs-volume-plugin/README.md b/s3fs-volume-plugin/README.md index 4b19990..2682820 100644 --- a/s3fs-volume-plugin/README.md +++ b/s3fs-volume-plugin/README.md @@ -9,7 +9,7 @@ This is a managed Docker volume plugin to allow Docker containers to access S3Fs - This is a managed plugin only, no legacy support. - In order to properly support versions use `--alias` when installing the plugin. - This only supports one S3Fs cluster per instance use `--alias` to define separate instances -- The value of `AWSACCESSKEYID/AWSSECRETACCESSKEY` is initially blank it needs `docker plugin s3fs set AWSACCESSKEYID=key;docker plugin s3fs set AWSSECRETACCESSKEY=secret` if it is set then it will be used for all servers and low level options will not be allowed. Primarily this is to control what the deployed stacks can perform. +- The value of `AWSACCESSKEYID/AWSSECRETACCESSKEY` is initially blank it needs `docker plugin s3fs set AWSACCESSKEYID=key;docker plugin s3fs set AWSSECRETACCESSKEY=secret` if it is set then it will be used for all buckets and low level options will not be allowed. Primarily this is to control what the deployed stacks can perform. - **There is no robust error handling. So garbage in -> garbage out** ## Operating modes @@ -18,7 +18,7 @@ There are three operating modes listed in order of preference. Each are mutuall ### Just the name -This is the *recommended* approach for production systems as it will prevent stacks from specifying any random server. It also prevents the stack configuration file from containing environment specific servers and instead defers that knowledge to the plugin only which is set on the node level. This relies on `SERVERS` being configured and will use the name as the volume mount set by [`docker plugin set`](https://docs.docker.com/engine/reference/commandline/plugin_set/). This can be done in an automated fashion as: +This is the *recommended* approach for production systems as it will prevent stacks from specifying any random server. It also prevents the stack configuration file from containing environment specific key/secrets and instead defers that knowledge to the plugin only which is set on the node level. This relies on `AWSACCESSKEYID/AWSSECRETACCESSKEY` being configured and will use the name as the volume mount set by [`docker plugin set`](https://docs.docker.com/engine/reference/commandline/plugin_set/). This can be done in an automated fashion as: docker plugin install --alias PLUGINALIAS \ trajano/s3fs-volume-plugin \ @@ -27,7 +27,7 @@ This is the *recommended* approach for production systems as it will prevent sta docker plugin set PLUGINALIAS AWSSECRETACCESSKEY=secret docker plugin enable PLUGINALIAS -If there is a need to have a different set of servers, a separate plugin alias should be created with a different set of servers. +If there is a need to have a different set of key/secrets, a separate plugin alias should be created with a different set of key/secrets. Example in docker-compose.yml: @@ -40,7 +40,7 @@ The `volumes.x.name` specifies the bucket and optionally a subdirectory mount. ### Specify the s3fs driver opts -This uses the `driver_opts.servers` to define a comma separated list s3fs options. The rules for specifying the volume is the same as the previous section. +This uses the `driver_opts.s3fsopts` to define a comma separated list s3fs options. The rules for specifying the volume is the same as the previous section. Example in docker-compose.yml assuming the alias was set as `s3fs`: @@ -57,7 +57,7 @@ The `volumes.x.name` specifies the bucket and optionally a subdirectory mount. ### Specify the options -This passes the `driver_opts.glusteropts` to the `s3fs` command followed by the generated mount point. This is the most flexible method and gives full range to the options of the S3Fs FUSE client. Example in docker-compose.yml assuming the alias was set as `s3fs`: +This passes the `driver_opts.s3fsopts` to the `s3fs` command followed by the generated mount point. This is the most flexible method and gives full range to the options of the S3Fs FUSE client. Example in docker-compose.yml assuming the alias was set as `s3fs`: volumes: sample: