Provided base ceph-rbd CSI driver example (#8664)

demo/csi/ceph-csi-plugin/README.md

@@ -0,0 +1,66 @@
+# Openstack Ceph-CSI Plugin
+
+The configuration here is for the Ceph RBD driver, migrated from the k8s config [documentation](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-rbd.md). It can be easily modified for the CephFS Driver, as used [here](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-cephfs.md).
+
+## Requirements
+
+The example plugin job creates a file at `local/cloud.conf` using a [`template`](https://www.nomadproject.io/docs/job-specification/template) stanza which pulls the necessary credentials from a [Vault kv-v2](https://www.vaultproject.io/docs/secrets/kv/kv-v2) secrets store. 
+
+
+### Docker Privileged Mode
+
+The Ceph CSI Node task requires that [`privileged = true`](https://www.nomadproject.io/docs/drivers/docker#privileged) be set. This is not needed for the Controller task.
+
+## Container Arguments
+
+Refer to the official plugin [guide](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-rbd.md).
+
+- `--type=rbd`
+
+  - Driver type `rbd` (or alternately `cephfs`)
+
+- `--endpoint=unix:///csi/csi.sock`
+
+  - This option must match the `mount_dir` specified in the `csi_plugin` stanza for the task.
+
+- `--nodeid=${node.unique.name}`
+
+  - A unique ID for the node the task is running on. Recommend using `${node.unique.name}`
+
+- `--cluster=${NOMAD_DC}`
+
+  - The cluster the Controller/Node is a part of. Recommend using `${NOMAD_DC}`
+
+- `--instanceid=${attr.unique.platform.aws.instance-id}`
+
+  - Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning. Used for topology-aware deployments.
+
+## Deployment
+
+### Plugin
+
+```bash
+export NOMAD_ADDR=https://nomad.example.com:4646
+export NOMAD_TOKEN=34534-3sdf3-szfdsafsdf3423-zxdfsd3
+nomad job run ceph-csi-plugin.hcl
+```
+
+### Volume Registration
+
+The `external_id` value for the volume must be strictly formatted, see `ceph_csi.tf`. Based on [Ceph-CSI ID Format](https://github.com/ceph/ceph-csi/blob/71ddf51544be498eee03734573b765eb04480bb9/internal/util/volid.go#L27), see [examples](https://github.com/ceph/ceph-csi/blob/71ddf51544be498eee03734573b765eb04480bb9/internal/util/volid_test.go#L33).
+
+The `secrets` block will be populated with values pulled from `/etc/ceph/ceph.client.<user>.keyring`, e.g.
+```
+userid = "<user>"
+userkey = "AWBg/BtfJInSFBATOrrnCh6UGE3QB3nYakdF+g=="
+```
+
+```bash
+export NOMAD_ADDR=https://nomad.example.com:4646
+export NOMAD_TOKEN=34534-3sdf3-szfdsafsdf3423-zxdfsd3
+nomad volume register example_volume.hcl
+```
+
+## Ceph CSI Driver Source
+
+- https://github.com/ceph/ceph-csi

demo/csi/ceph-csi-plugin/ceph-csi-id.tf

@@ -0,0 +1,33 @@
+locals {
+  # ClusterID: Is a unique ID per cluster that the CSI instance is serving and is restricted to
+  # lengths that can be accommodated in the encoding scheme.
+  # must be less than 128 chars. must match the cluster id in the csi plugin conf.
+  ClusterID = "<clusterid>"
+
+  # EncodingVersion: Carries the version number of the encoding scheme used to encode the CSI ID,
+  # and is preserved for any future proofing w.r.t changes in the encoding scheme, and to retain
+  # ability to parse backward compatible encodings.
+  # https://github.com/ceph/ceph-csi/blob/ef1785ce4db0aa1f6878c770893bcabc71cff300/internal/cephfs/driver.go#L31
+  EncodingVersion = 1
+
+  # LocationID: 64 bit integer identifier determining the location of the volume on the Ceph cluster.
+  # It is the ID of the poolname or fsname, for RBD or CephFS backed volumes respectively.
+  # see https://docs.ceph.com/docs/mimic/rbd/rados-rbd-cmds/
+  LocationID = 7
+
+  # ObjectUUID: Is the on-disk uuid of the object (image/snapshot) name, for the CSI volume that
+  # corresponds to this CSI ID.. must be 36 chars long.
+  ObjectUUID = "abcd"
+}
+
+data "template_file" "csi_id" {
+  template = "$${versionEncodedHex}-$${clusterIDLength}-$${ciClusterID}-$${poolIDEncodedHex}-$${ciObjectUUID}"
+
+  vars = {
+    versionEncodedHex = "${format("%02X", local.EncodingVersion)}"
+    clusterIDLength   = "${format("%02X", length(local.ClusterID))}"
+    ciClusterID       = "${local.ClusterID}"
+    poolIDEncodedHex  = "${format("%016X", local.LocationID)}"
+    ciObjectUUID      = "${local.ObjectUUID}"
+  }
+}

demo/csi/ceph-csi-plugin/ceph-csi-plugin.hcl

@@ -0,0 +1,119 @@
+job "ceph-csi-plugin" {
+  datacenters = ["dc1"]
+  type        = "system"
+  group "nodes" {
+    task "ceph-node" {
+      driver = "docker"
+      template {
+        data        = <<EOF
+[{
+    "clusterID": "<clusterid>",
+    "monitors": [
+        {{range $index, $service := service "mon.ceph"}}{{if gt $index 0}}, {{end}}"{{.Address}}"{{end}}
+    ]
+}]
+EOF
+        destination = "local/config.json"
+        change_mode = "restart"
+      }
+      config {
+        image = "quay.io/cephcsi/cephcsi:v2.1.2-amd64"
+        volumes = [
+          "./local/config.json:/etc/ceph-csi-config/config.json"
+        ]
+        mounts = [
+          {
+            type     = "tmpfs"
+            target   = "/tmp/csi/keys"
+            readonly = false
+            tmpfs_options {
+              size = 1000000 # size in bytes
+            }
+          }
+        ]
+        args = [
+          "--type=rbd",
+          # Name of the driver
+          "--drivername=rbd.csi.ceph.com",
+          "--logtostderr",
+          "--nodeserver=true",
+          "--endpoint=unix://csi/csi.sock",
+          "--instanceid=${attr.unique.platform.aws.instance-id}",
+          "--nodeid=${attr.unique.consul.name}",
+          # TCP port for liveness metrics requests (/metrics)
+          "--metricsport=${NOMAD_PORT_prometheus}",
+        ]
+        privileged = true
+        resources {
+          cpu    = 200
+          memory = 500
+          network {
+            mbits = 1
+            // prometheus metrics port
+            port "prometheus" {}
+          }
+        }
+      }
+      service {
+        name = "prometheus"
+        port = "prometheus"
+        tags = ["ceph-csi"]
+      }
+      csi_plugin {
+        id        = "ceph-csi"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+    }
+    task "ceph-controller" {
+
+      template {
+        data        = <<EOF
+[{
+    "clusterID": "<clusterid>",
+    "monitors": [
+        {{range $index, $service := service "mon.ceph"}}{{if gt $index 0}}, {{end}}"{{.Address}}"{{end}}
+    ]
+}]
+EOF
+        destination = "local/config.json"
+        change_mode = "restart"
+      }
+      driver = "docker"
+      config {
+        image = "quay.io/cephcsi/cephcsi:v2.1.2-amd64"
+        volumes = [
+          "./local/config.json:/etc/ceph-csi-config/config.json"
+        ]
+        resources {
+          cpu    = 200
+          memory = 500
+          network {
+            mbits = 1
+            // prometheus metrics port
+            port "prometheus" {}
+          }
+        }
+        args = [
+          "--type=rbd",
+          "--controllerserver=true",
+          "--drivername=rbd.csi.ceph.com",
+          "--logtostderr",
+          "--endpoint=unix://csi/csi.sock",
+          "--metricsport=$${NOMAD_PORT_prometheus}",
+          "--nodeid=$${attr.unique.platform.aws.hostname}"
+        ]
+      }
+      service {
+        name = "prometheus"
+        port = "prometheus"
+        tags = ["ceph-csi"]
+      }
+      csi_plugin {
+        id        = "ceph-csi"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+    }
+  }
+}

demo/csi/ceph-csi-plugin/example_volume.hcl

@@ -0,0 +1,22 @@
+type = "csi"
+id   = "testvol"
+name = "test_volume"
+# this must be strictly formatted, see README
+external_id     = "ffff-0024-01616094-9d93-4178-bf45-c7eac19e8b15-000000000000ffff-00000000-1111-2222-bbbb-cacacacacaca"
+access_mode     = "single-node-writer"
+attachment_mode = "block-device"
+plugin_id       = "ceph-csi"
+mount_options {
+  fs_type = "ext4"
+}
+parameters {}
+secrets {
+  userID  = "<userid>"
+  userKey = "<userkey>"
+}
+context {
+  # note: although these are 'parameters' in the ceph-csi spec
+  # they are passed through to the provider as 'context'
+  clusterID = "<clusterid>"
+  pool      = "my_pool"
+}