diff --git a/playbooks/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml b/playbooks/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml index 315d908d297cb..eb4e2926b39b2 100644 --- a/playbooks/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml +++ b/playbooks/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml @@ -1,37 +1,28 @@ - name: Set up Kubernetes local cluster hosts: all roles: - - install-k8s-jobs-dependences + - export-vexxhost-openrc become: yes tasks: - name: Set up Kubernetes local cluster shell: cmd: | + set -x set -e + set -o pipefail + apt-get install python-pip -y pip install -U python-openstackclient - export OS_DOMAIN_NAME=$(echo '{{ vexxhost_credentials.user_domain_name }}') - export OS_AUTH_TYPE=$(echo '{{ vexxhost_credentials.auth_type }}') - export OS_IDENTITY_API_VERSION=$(echo '{{ vexxhost_credentials.identity_api_version }}') - export OS_VOLUME_API_VERSION=$(echo '{{ vexxhost_credentials.volume_api_version }}') - export OS_INTERFACE=$(echo '{{ vexxhost_credentials.interface }}') - export OS_AUTH_URL=$(echo '{{ vexxhost_credentials.auth_url }}') - export OS_PROJECT_ID=$(echo '{{ vexxhost_credentials.project_id }}') - export OS_PROJECT_NAME=$(echo '{{ vexxhost_credentials.project_name }}') - export OS_USER_DOMAIN_NAME=$(echo '{{ vexxhost_credentials.user_domain_name }}') - export OS_PROJECT_DOMAIN_ID=$(echo '{{ vexxhost_credentials.project_domain_id }}') - export OS_USERNAME=$(echo '{{ vexxhost_credentials.username }}') - export OS_PASSWORD=$(echo '{{ vexxhost_credentials.password }}') - export OS_REGION_NAME=$(echo '{{ vexxhost_credentials.region_name }}') + mkdir -p /etc/kubernetes/ + cp ./examples/webhook/policy.json /etc/kubernetes/ + sed -i "s/c1f7910086964990847dc6c8b128f63c/$OS_PROJECT_ID/g" /etc/kubernetes/policy.json + sed -i -e "s/k8s-admin/creator/g" /etc/kubernetes/policy.json - if [[ ! -d "/etc/kubernetes/" ]]; then - sudo mkdir -p /etc/kubernetes/ - fi - chown zuul /etc/kubernetes/ + # Create cloud-config cat << EOF >> /etc/kubernetes/cloud-config [Global] - domain-name = ${OS_PROJECT_DOMAIN_NAME-$OS_PROJECT_DOMAIN_ID} + domain-name = $OS_USER_DOMAIN_NAME tenant-id = $OS_PROJECT_ID auth-url = $OS_AUTH_URL password = $OS_PASSWORD @@ -39,8 +30,10 @@ region = $OS_REGION_NAME [BlockStorage] bs-version = v2 + ignore-volume-az = yes EOF + # Create webhook.kubeconfig cat << EOF >> /etc/kubernetes/webhook.kubeconfig apiVersion: v1 clusters: @@ -60,60 +53,34 @@ - name: webhook EOF - set -x - make depend - make build - mkdir -p "{{ ansible_user_dir }}/.kube" - export API_HOST_IP="172.17.0.1" + # Go where we cloned kubernetes repository + cd $GOPATH/src/k8s.io/kubernetes/ + export API_HOST_IP=$(ifconfig | awk '/^docker0/ {getline; print $2}' | awk -F ':' '{print $2}') export KUBELET_HOST="0.0.0.0" - - echo "Stopping firewall and allow all traffic..." - iptables -F - iptables -X - iptables -t nat -F - iptables -t nat -X - iptables -t mangle -F - iptables -t mangle -X - iptables -P INPUT ACCEPT - iptables -P FORWARD ACCEPT - iptables -P OUTPUT ACCEPT export ALLOW_SECURITY_CONTEXT=true export ENABLE_CRI=false export ENABLE_HOSTPATH_PROVISIONER=true export ENABLE_SINGLE_CA_SIGNER=true - # export KUBE_ENABLE_CLUSTER_DASHBOARD=true export KUBE_ENABLE_CLUSTER_DNS=false - export LOG_LEVEL=10 - # we want to use the openstack cloud provider + export LOG_LEVEL=4 + # We want to use the openstack cloud provider export CLOUD_PROVIDER=openstack - # we want to run a separate cloud-controller-manager for openstack + # We want to run a separate cloud-controller-manager for openstack export EXTERNAL_CLOUD_PROVIDER=true # DO NOT change the location of the cloud-config file. It is important for the old cinder provider to work export CLOUD_CONFIG=/etc/kubernetes/cloud-config - # specify the OCCM binary - export EXTERNAL_CLOUD_PROVIDER_BINARY="{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/openstack-cloud-controller-manager" - # Cleanup some directories just in case - sudo rm -rf /var/lib/kubelet/* + # Specify the OCCM binary + export EXTERNAL_CLOUD_PROVIDER_BINARY='{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/openstack-cloud-controller-manager' # location of where the kubernetes processes log their output - mkdir -p "{{ ansible_user_dir }}/workspace/logs/kubernetes" - export LOG_DIR="{{ ansible_user_dir }}/workspace/logs/kubernetes" + mkdir -p '{{ ansible_user_dir }}/workspace/logs/kubernetes' + export LOG_DIR='{{ ansible_user_dir }}/workspace/logs/kubernetes' # We need this for one of the conformance tests export ALLOW_PRIVILEGED=true # Just kick off all the processes and drop down to the command line export ENABLE_DAEMON=true - # We need the hostname to match the name of the vm started by openstack export HOSTNAME_OVERRIDE=$(curl http://169.254.169.254/openstack/latest/meta_data.json | python -c "import sys, json; print json.load(sys.stdin)['name']") - - # copy the same policy json and fix up the hard coded project id - cp ./examples/webhook/policy.json /etc/kubernetes/ - sed -i -e "s|c1f7910086964990847dc6c8b128f63c|$OS_PROJECT_ID|g" /etc/kubernetes/policy.json - # pick an existing role of the user and replace the k8s-admin role in policy.json - sed -i -e "s|k8s-admin|creator|g" /etc/kubernetes/policy.json - # print the modified file - cat /etc/kubernetes/policy.json - - pushd ${GOPATH}/src/k8s.io/kubernetes + export MAX_TIME_FOR_URL_API_SERVER=5 export AUTHORIZATION_MODE="Node,Webhook,RBAC" sed -i -e "/kube::util::wait_for_url.*$/,+1d" hack/local-up-cluster.sh @@ -122,18 +89,13 @@ # -E preserves the current env vars, but we need to special case PATH sudo -E PATH=$PATH SHELLOPTS=$SHELLOPTS ./hack/local-up-cluster.sh -O - nohup "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/k8s-keystone-auth" \ --tls-cert-file /var/run/kubernetes/serving-kube-apiserver.crt \ --tls-private-key-file /var/run/kubernetes/serving-kube-apiserver.key \ --keystone-policy-file /etc/kubernetes/policy.json \ --log-dir=${LOG_DIR} \ --v=10 \ - --keystone-url ${OS_AUTH_URL} >"${LOG_DIR}/keystone-auth.log" 2>&1 & - - # sudo of local-up-cluster mucks with permissions - sudo chmod -R 777 "{{ ansible_user_dir }}/.kube" - sudo chmod 777 /var/run/kubernetes/client-admin.key + --keystone-url ${OS_AUTH_URL} >"${LOG_DIR}/keystone-auth.log" 2>&1 & # set up the config we need for kubectl to work cluster/kubectl.sh config set-cluster local --server=https://localhost:6443 --certificate-authority=/var/run/kubernetes/server-ca.crt @@ -150,41 +112,47 @@ cluster/kubectl.sh create clusterrolebinding --user system:kube-controller-manager kube-system-cluster-admin-6 --clusterrole cluster-admin { - TOKEN=$(openstack token issue -f value -c id) - authenticated_info=`cat << EOF | curl -kvs -XPOST -d @- https://localhost:8443/webhook | python -c "import sys, json; print json.load(sys.stdin)" - { - "apiVersion": "authentication.k8s.io/v1beta1", - "kind": "TokenReview", - "metadata": { - "creationTimestamp": null - }, - "spec": { - "token": "$TOKEN" - } - } - EOF` - base_body=`cat << EOF | python -c "import sys, json; print json.load(sys.stdin)" - { - "apiVersion": "authorization.k8s.io/v1beta1", - "kind": "SubjectAccessReview", - "spec": { - "resourceAttributes": { - "namespace": "default", - "verb": "get", - "group": "", - "resource": "pods" - } - } - } - EOF` - authorization_body=$(python -c "import json; s1=${authenticated_info}; s2=${base_body}; \ - s2['spec']['user']=s1['status']['user']['username']; \ - s2['spec']['group']=s1['status']['user']['groups']; \ - s2['spec']['extra']=s1['status']['user']['extra'];print json.dumps(s2)") - allowed=$(echo $authorization_body | curl -kvs -XPOST -d @- https://localhost:8443/webhook | python -mjson.tool) - } 1> /dev/null 2>&1 - echo ${allowed} - [[ "${allowed}" =~ '"allowed": true' ]] && echo "Testing k8s-keystone-auth sucessfully!" + authenticated_info=$(cat <<< ' + { + "apiVersion": "authentication.k8s.io/v1beta1", + "kind": "TokenReview", + "metadata": { + "creationTimestamp": null + }, + "spec": { + "token": "'$(openstack token issue -f value -c id)'" + } + }' | curl -kvs -XPOST -d @- https://localhost:8443/webhook | python -c "import sys, json; print json.load(sys.stdin)" + ) + base_body=$(cat <<< ' + { + "apiVersion": "authorization.k8s.io/v1beta1", + "kind": "SubjectAccessReview", + "spec": { + "resourceAttributes": { + "namespace": "default", + "verb": "get", + "group": "", + "resource": "pods" + } + } + }' | python -c "import sys, json; print json.load(sys.stdin)" + ) + update_auth_info=$(echo " + import json; + s1=$authenticated_info; + s2=$base_body; + s2['spec']['user']=s1['status']['user']['username']; + s2['spec']['group']=s1['status']['user']['groups']; + s2['spec']['extra']=s1['status']['user']['extra']; + print json.dumps(s2)" | sed 's/^ \+//' + ) + authorization_body=$(python -c "$update_auth_info") + allowed=$(echo "$authorization_body" | curl -kvs -XPOST -d @- https://localhost:8443/webhook | python -mjson.tool) + } > /dev/null 2>&1 + + echo "$allowed" + [[ "$allowed" =~ '"allowed": true' ]] && echo "Testing k8s-keystone-auth sucessfully!" cluster/kubectl.sh config set-credentials openstackuser --auth-provider=openstack cluster/kubectl.sh config set-context --cluster=local --user=openstackuser openstackuser@local @@ -193,7 +161,6 @@ echo "Testing kubernetes+keystone authentication and authorizatio failed!" exit 1 fi - popd executable: /bin/bash chdir: '{{ zuul.project.src_dir }}' - environment: '{{ golang_env }}' + environment: '{{ golang_env | combine(vexxhost_openrc) }}' diff --git a/roles/export-vexxhost-openrc/tasks/main.yaml b/roles/export-vexxhost-openrc/tasks/main.yaml index 042177428597b..25791da9bd3a1 100644 --- a/roles/export-vexxhost-openrc/tasks/main.yaml +++ b/roles/export-vexxhost-openrc/tasks/main.yaml @@ -13,3 +13,4 @@ OS_USERNAME: '{{ vexxhost_credentials.username }}' OS_PASSWORD: '{{ vexxhost_credentials.password }}' OS_REGION_NAME: '{{ vexxhost_credentials.region_name }}' + OS_DOMAIN_NAME: '{{ vexxhost_credentials.user_domain_name }}' diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 90aa02bc19d08..75fe0997eb8c5 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -300,11 +300,9 @@ - job: name: cloud-provider-openstack-acceptance-test-keystone-authentication-authorization - parent: golang-test + parent: cloud-provider-openstack-acceptance-test description: | - Run acceptance tests of cloud-provider-openstack repo of K8S+keystone authentication and - authorization scenario against vexxhost cloud + Run keystone auth acceptance tests of cloud-provider-openstack run: playbooks/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml secrets: - vexxhost_credentials - nodeset: ubuntu-xenial-vexxhost