Merge pull request apache#160 from animationzl/add-cinder-flexvolume

Add job cloud-provider-openstack-acceptance-test-flexvolume-cinder

playbooks/cloud-provider-openstack-acceptance-test-flexvolume-cinder/run.yaml

@@ -0,0 +1,117 @@
+- name: Run k8s flex volume cinder test
+  hosts: all
+  become: yes
+  pre_tasks:
+    - shell:
+        cmd: |
+          apt install libvirt-bin -y
+        executable: /bin/bash
+  roles:
+    - clone-devstack-gate-to-workspace
+    - role: create-devstack-local-conf
+      enable_services:
+        - 'ceph'
+    - role: install-devstack
+      environment:
+        OVERRIDE_ENABLED_SERVICES: 'key,c-sch,c-api,c-vol,rabbit,mysql'
+        PROJECTS: 'openstack/devstack-plugin-ceph'
+  tasks:
+    - shell:
+        cmd: |
+          set -x
+          set -e
+          set -o pipefail
+
+          # Build cloud-provider-openstack binaries
+          make build
+
+          # Create cloud-config
+          source /opt/stack/new/devstack/openrc admin admin
+          mkdir -p /etc/kubernetes/
+          cat << EOF >> /etc/kubernetes/cloud-config
+          [Global]
+          domain-id = $OS_PROJECT_DOMAIN_ID
+          tenant-name = $OS_PROJECT_NAME
+          auth-url = $OS_AUTH_URL
+          password = $OS_PASSWORD
+          username = $OS_USERNAME
+          region = $OS_REGION_NAME
+
+          [BlockStorage]
+          bs-version = v2
+          ignore-volume-az = yes
+          EOF
+
+          export API_HOST_IP=$(ip route get 1.1.1.1 | awk '{print $7}')
+          export KUBELET_HOST="0.0.0.0"
+          export ALLOW_SECURITY_CONTEXT=true
+          export ENABLE_CRI=false
+          export ENABLE_HOSTPATH_PROVISIONER=true
+          export ENABLE_SINGLE_CA_SIGNER=true
+          export KUBE_ENABLE_CLUSTER_DNS=false
+          export LOG_LEVEL=4
+          # DO NOT change the location of the cloud-config file. It is important for the old cinder provider to work
+          export CLOUD_CONFIG=/etc/kubernetes/cloud-config
+
+          # location of where the kubernetes processes log their output
+          mkdir -p '{{ ansible_user_dir }}/workspace/logs/kubernetes'
+          export LOG_DIR='{{ ansible_user_dir }}/workspace/logs/kubernetes'
+          # We need this for one of the conformance tests
+          export ALLOW_PRIVILEGED=true
+          # Just kick off all the processes and drop down to the command line
+          export ENABLE_DAEMON=true
+          export MAX_TIME_FOR_URL_API_SERVER=5
+          export RUNTIME_CONFIG="settings.k8s.io/v1alpha1=true"
+
+          mkdir -p /usr/libexec/kubernetes/kubelet-plugins/volume/exec/cinder.io~cinder-flex-volume-driver/
+          cp cinder-flex-volume-driver /usr/libexec/kubernetes/kubelet-plugins/volume/exec/cinder.io~cinder-flex-volume-driver/
+
+          export k8s_root="$GOPATH/src/k8s.io/kubernetes"
+          export kubectl="$k8s_root/cluster/kubectl.sh"
+          # -E preserves the current env vars, but we need to special case PATH
+          sudo -E PATH=$PATH SHELLOPTS=$SHELLOPTS "$k8s_root/hack/local-up-cluster.sh" -O
+
+          # set up the config we need for kubectl to work
+          "$kubectl" config set-cluster local --server=https://localhost:6443 --certificate-authority=/var/run/kubernetes/server-ca.crt
+          "$kubectl" config set-credentials myself --client-key=/var/run/kubernetes/client-admin.key --client-certificate=/var/run/kubernetes/client-admin.crt
+          "$kubectl" config set-context local --cluster=local --user=myself
+          "$kubectl" config use-context local
+
+          # Hack for RBAC for all for the new cloud-controller process, we need to do better than this
+          "$kubectl" create clusterrolebinding --user system:serviceaccount:kube-system:default kube-system-cluster-admin-1 --clusterrole cluster-admin
+          "$kubectl" create clusterrolebinding --user system:serviceaccount:kube-system:pvl-controller kube-system-cluster-admin-2 --clusterrole cluster-admin
+          "$kubectl" create clusterrolebinding --user system:serviceaccount:kube-system:cloud-node-controller kube-system-cluster-admin-3 --clusterrole cluster-admin
+          "$kubectl" create clusterrolebinding --user system:serviceaccount:kube-system:cloud-controller-manager kube-system-cluster-admin-4 --clusterrole cluster-admin
+          "$kubectl" create clusterrolebinding --user system:serviceaccount:kube-system:shared-informers kube-system-cluster-admin-5 --clusterrole cluster-admin
+          "$kubectl" create clusterrolebinding --user system:kube-controller-manager  kube-system-cluster-admin-6 --clusterrole cluster-admin
+
+          # Avoid missing feature 400000000000000 error in ceph
+          # http://lists.ceph.com/pipermail/ceph-users-ceph.com/2017-July/019383.html
+          ceph osd crush tunables hammer
+
+          # Sleep to wait for creating serviceaccount default/default complete
+          sleep 5
+          # Run test
+          export VOLUME_ID=$(cinder create 1 | awk '/ id / {print $4}')
+          "$kubectl" create -f examples/cinder-flexvolume/nginx.yaml
+
+          # If test passed
+          if timeout 300 bash -c '
+              while :
+              do
+                  [[ $("$kubectl" describe pod nginx | awk "/^Status:/ {print \$2}") == Running ]] && break
+                  sleep 1
+              done
+              '
+          then
+              echo 'Run test successfully'
+              "$kubectl" get pod nginx
+          else
+              echo 'Run test failed'
+              "$kubectl" get pod nginx
+              "$kubectl" describe pod nginx
+              exit 1
+          fi
+        executable: /bin/bash
+        chdir: '{{ zuul.project.src_dir }}'
+      environment: '{{ golang_env }}'

roles/create-devstack-local-conf/tasks/main.yml

@@ -149,3 +149,22 @@
   environment: '{{ zuul | zuul_legacy_vars }}'
   when:
     - '"zun" in enable_services'
+
+- name: create devstack local conf with ceph enabled
+  shell:
+    cmd: |
+      set -e
+      set -x
+      cat << EOF >> /tmp/dg-local.conf
+      enable_plugin devstack-plugin-ceph https://git.openstack.org/openstack/devstack-plugin-ceph refs/changes/70/561370/3
+      ENABLE_CEPH_CINDER=True
+      ENABLE_CEPH_GLANCE=False
+      ENABLE_CEPH_C_BAK=False
+      ENABLE_CEPH_NOVA=False
+      ENABLE_CEPH_MANILA=False
+      EOF
+    executable: /bin/bash
+    chdir: '{{ ansible_user_dir }}/workspace'
+  environment: '{{ zuul | zuul_legacy_vars }}'
+  when:
+    - '"ceph" in enable_services'

zuul.d/jobs.yaml

@@ -316,3 +316,11 @@
     run: playbooks/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml
     secrets:
       - vexxhost_credentials
+
+- job:
+    name: cloud-provider-openstack-acceptance-test-flexvolume-cinder
+    parent: cloud-provider-openstack-acceptance-test
+    description: |
+      Run cinder flexvolume acceptance tests of cloud-provider-openstack
+    run: playbooks/cloud-provider-openstack-acceptance-test-flexvolume-cinder/run.yaml
+    nodeset: ubuntu-xenial