diff --git a/LICENSE b/LICENSE index 5b08cc7..e695c85 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014, Quobyte Inc. +Copyright (c) 2017, Quobyte Inc. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index 2c37993..5263ea5 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,10 @@ -Scripts and tools for deploying Quobyte installations +Scripts and tools for deploying Quobyte on Kubernetes ===================================================== Currently contains: - * **Ansible** script for AWS deployments + * **demo** Vagrant file for local demo cluster and kubespray based k8s cluster bootstrap * device initialization tools: **qbootstrap** and **qmkdev** - * **Kubernetes** specification files for Quobyte on Kubernetes - -For **Puppet**, please refer to SysEleven's recipes: -https://github.com/syseleven/puppet-quobyte - -Arnold from Inovex maintains a **Saltstack** formula here: -https://github.com/bechtoldt/saltstack-quobyte-formula - -For **Mesos**, please check out our Mesos framework: -https://github.com/quobyte/mesos-framework + * **deploy** Specification files for Quobyte on Kubernetes For automated **Kubernetes** deployments checkout Quobyte Deployer (community tool): https://github.com/johscheuer/quobyte-kubernetes-operator diff --git a/ansible/aws/ansible.cfg b/ansible/aws/ansible.cfg deleted file mode 100644 index 2a19e21..0000000 --- a/ansible/aws/ansible.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[defaults] -private_key_file=key.pem -# Default user on AWS, change to any user that has sudo permissions. -remote_user=ubuntu - -[paramiko] -record_host_keys=True - diff --git a/ansible/aws/host.cfg b/ansible/aws/host.cfg deleted file mode 100644 index 7c69255..0000000 --- a/ansible/aws/host.cfg +++ /dev/null @@ -1,2 +0,0 @@ -# Address(es) of Quobyte registry services -dir_service={{ registry_server }} diff --git a/ansible/aws/hosts b/ansible/aws/hosts deleted file mode 100644 index 22d6e90..0000000 --- a/ansible/aws/hosts +++ /dev/null @@ -1,35 +0,0 @@ -# This playbook sets up a minimal installation with replication. As such it needs -# at least three servers with all device types enabled. -# It does not setup authentication and must therefore be run in a trusted environment. -# -# Each line names one host along with variables in the format key=value -# Supported variables are: -# - bootstrap_device= -# - dir_device= -# - metadata_device= -# - data_device= -# -# For demo set-ups on machines (or VMs) with a single drive, use the test_devices=yes -# parameter. This will create three bind mounts on the machine at /test_dev_{d,r,m} -# which can be used for the variables configuring device paths. -# -# After running the playbook, you can reach the webconsole at: -# http://server1:8080 -# -# qmgmt can be executed on any machine that can connect to server1 with: -# qmgmt -u http://server1 ... -# -# Volumes can be mounted with the following command: -# mount.quobyte server1,server2,server3/ ~/mymountpoint - -[machines] -# Minimal example: -# server1 bootstrap_device=/mnt/data0 metadata_device=/mnt/data1 data_device=/mnt/data2 -# server2 registry_device=/mnt/data0 metadata_device=/mnt/data1 data_device=/mnt/data2 -# server3 registry_device=/mnt/data0 metadata_device=/mnt/data1 data_device=/mnt/data2 - -[machines:vars] -## Repository URL for your distribution, for example: -repository=https://support.quobyte.com/repo/1//xUbuntu_14.04 -## Repeat all servers that have bootstrap_device or registry_device set, for example: -registry_server=server1:50001,server2:50001,server3:50001 diff --git a/ansible/aws/playbook.yml b/ansible/aws/playbook.yml deleted file mode 100644 index 6870803..0000000 --- a/ansible/aws/playbook.yml +++ /dev/null @@ -1,111 +0,0 @@ ---- -- hosts: machines - sudo: yes - tasks: - - name: Copy JRE8 package - copy: src=jre_8-2_all.deb dest=/tmp/jre_8-2_all.deb - - - name: Install JRE8 - apt: deb=/tmp/jre_8-2_all.deb - - - name: setup ppa repository on Ubuntu - apt_repository: repo="ppa:webupd8team/java" - - - name: Import Quobyte USP repo key - apt_key: url={{repository}}/Release.key state=present - - - name: setup repository on Ubuntu - apt_repository: repo="deb {{repository}} ./" state=present - - - name: install services on Ubuntu - apt: name=quobyte-usp-server state=present update_cache=yes - - - name: install client and tools on Ubuntu - apt: name=quobyte-usp-client state=present update_cache=yes - - - name: Assures /etc/quobyte dir exists - file: path=/etc/quobyte state=directory - - - name: install host.cfg - template: src=host.cfg dest=/etc/quobyte/host.cfg - - - name: create bind test mountpoints - file: path=/{{ item }} state=directory - with_items: - - /test_dev_r - - /test_dev_m - - /test_dev_d - when: test_devices is defined - - - name: create bind test data dirs - file: path=/{{ item }} state=directory - with_items: - - /tmp/test_dev_r - - /tmp/test_dev_m - - /tmp/test_dev_d - when: test_devices is defined - - - name: Check for test device mounts - command: grep '/test_dev_' /proc/mounts - register: grep_temp_mount - always_run: true - failed_when: grep_temp_mount.rc > 1 - - - name: Mount test devices with mount bind - shell: mount --bind /tmp{{ item }} {{ item }} - with_items: - - /test_dev_r - - /test_dev_m - - /test_dev_d - when: test_devices is defined and grep_temp_mount.rc == 1 - - - name: Create bootstrap device - shell: qbootstrap -y {{bootstrap_device}} creates={{bootstrap_device}}/QUOBYTE_DEV_SETUP - when: bootstrap_device is defined - - - name: Start bootstrap registry service - service: name=quobyte-dir state=started enabled=yes - when: bootstrap_device is defined - - - name: Start Web console - service: name=quobyte-webconsole state=started enabled=yes - when: bootstrap_device is defined - - - name: Start API services - service: name=quobyte-api state=started enabled=yes - when: bootstrap_device is defined - - - name: Prepare other registry devices - shell: qmkdev -t REGISTRY {{registry_device}} creates={{registry_device}}/QUOBYTE_DEV_SETUP - when: registry_device is defined - - - name: Start replicated registry services - service: name=quobyte-dir state=started enabled=yes - when: registry_device is defined - - - name: Wait for registry devices to come up - pause: minutes=1 - - - name: Add registry replica devices - shell: qmgmt registry add {{ item }} - with_items: - - 2 - - 3 - when: bootstrap_device is defined - - - - name: Initialize metadata devices - shell: qmkdev -t METADATA {{metadata_device}} creates={{metadata_device}}/QUOBYTE_DEV_SETUP - when: metadata_device is defined - - - name: Start metadata services - service: name=quobyte-mrc state=started enabled=yes - when: metadata_device is defined - - - name: Initialize data devices - shell: qmkdev -t DATA {{data_device}} creates={{data_device}}/QUOBYTE_DEV_SETUP - when: data_device is defined - - - name: Start data services - service: name=quobyte-osd state=started enabled=yes - when: data_device is defined diff --git a/demo/README.md b/demo/README.md new file mode 100644 index 0000000..6e2bf5d --- /dev/null +++ b/demo/README.md @@ -0,0 +1,60 @@ +## Setting up Test environment with Vagrant + +For a fast demo setup, we use a Vagrant based 4-machine cluster, where each server has additional 3 disk drives attached. + +```bash +$ cd examples/vagrant +$ vagrant up +$ vagrant ssh-config +``` + +We use kubespray to bootstrap and setup the Kubernetes cluster. +We provide an inventory file for the newly created cluster `demo/kubespray/inventory/vagrant`. +Please make sure that the *ansible_port* and *ansible_ssh_private_key_file* match. + + +If the 4 machines are running and you are able to connect to them like: +```bash +$ cd examples/vagrant +$ vagrant ssh qb1 +``` +we're good to apply some kubespray. + +```bash +$ cd examples/kubespray +$ ./clone_kubespray +$ ./ansible_cluster.sh +``` + +Make sure that `kubectl` [is installed](https://kubernetes.io/docs/tasks/tools/install-kubectl/ "Install and Set Up kubectl") on your machine. + +To configure and use your newly created cluster, you can run: + +```bash +$ mkdir -p $HOME/.kube/certs/qb +$ cd examples/vagrant/ +$ vagrant ssh qb1 -- -t sudo cat /etc/kubernetes/ssl/admin-qb1.pem > $HOME/.kube/certs/qb/qb-admin.pem +$ vagrant ssh qb1 -- -t sudo cat /etc/kubernetes/ssl/admin-qb1-key.pem > $HOME/.kube/certs/qb/qb-admin-key.pem +$ vagrant ssh qb1 -- -t sudo cat /etc/kubernetes/ssl/ca.pem > $HOME/.kube/certs/qb/qb-ca.pem + +$ kubectl config set-credentials qb-admin \ + --certificate-authority=$HOME/.kube/certs/qb/qb-ca.pem \ + --client-key=$HOME/.kube/certs/qb/qb-admin-key.pem \ + --client-certificate=$HOME/.kube/certs/qb/qb-admin.pem +$ kubectl config set-cluster qb --server=https://127.0.0.1:6443 \ + --certificate-authority=$HOME/.kube/certs/qb/qb-ca.pem + +$ kubectl config set-context qb --cluster=qb --user=qb-admin +$ kubectl config use-context qb +``` + +Your cluster should be available now: + +```bash +$ kubectl get nodes +NAME STATUS AGE VERSION +qb1 Ready 5m v1.7.3+coreos.0 +qb2 Ready 5m v1.7.3+coreos.0 +qb3 Ready 5m v1.7.3+coreos.0 +qb4 Ready 5m v1.7.3+coreos.0 +``` diff --git a/demo/kubespray/.gitignore b/demo/kubespray/.gitignore new file mode 100644 index 0000000..e09d31e --- /dev/null +++ b/demo/kubespray/.gitignore @@ -0,0 +1 @@ +kubespray diff --git a/demo/kubespray/ansible_cluster.sh b/demo/kubespray/ansible_cluster.sh new file mode 100755 index 0000000..348e24c --- /dev/null +++ b/demo/kubespray/ansible_cluster.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +ansible-playbook kubespray/cluster.yml -i inventory/vagrant -b -v diff --git a/demo/kubespray/ansible_reset.sh b/demo/kubespray/ansible_reset.sh new file mode 100755 index 0000000..432a421 --- /dev/null +++ b/demo/kubespray/ansible_reset.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +ansible-playbook kubespray/reset.yml -i inventory/vagrant -b -v diff --git a/demo/kubespray/ansible_upgrade.sh b/demo/kubespray/ansible_upgrade.sh new file mode 100755 index 0000000..c9fad42 --- /dev/null +++ b/demo/kubespray/ansible_upgrade.sh @@ -0,0 +1,3 @@ +!/bin/bash + +ansible-playbook kubespray/upgrade-cluster.yml -i inventory/vagrant -b -v diff --git a/demo/kubespray/clone_kubespray.sh b/demo/kubespray/clone_kubespray.sh new file mode 100755 index 0000000..44496f9 --- /dev/null +++ b/demo/kubespray/clone_kubespray.sh @@ -0,0 +1,3 @@ +#!/bin/bash +git clone https://github.com/kubernetes-incubator/kubespray.git + diff --git a/demo/kubespray/inventory/group_vars/all.yml b/demo/kubespray/inventory/group_vars/all.yml new file mode 100644 index 0000000..49ee844 --- /dev/null +++ b/demo/kubespray/inventory/group_vars/all.yml @@ -0,0 +1,90 @@ +## The access_ip variable is used to define how other nodes should access +## the node. This is used in flannel to allow other flannel nodes to see +## this node for example. The access_ip is really useful AWS and Google +## environments where the nodes are accessed remotely by the "public" ip, +## but don't know about that address themselves. +#access_ip: 1.1.1.1 + +### LOADBALANCING AND ACCESS MODES +## Enable multiaccess to configure etcd clients to access all of the etcd members directly +## as the "http://hostX:port, http://hostY:port, ..." and ignore the proxy loadbalancers. +## This may be the case if clients support and loadbalance multiple etcd servers natively. +#etcd_multiaccess: true + +## External LB example config +## apiserver_loadbalancer_domain_name: "elb.some.domain" +#loadbalancer_apiserver: +# address: 1.2.3.4 +# port: 1234 + +## Internal loadbalancers for apiservers +loadbalancer_apiserver_localhost: true + +## Local loadbalancer should use this port instead, if defined. +## Defaults to kube_apiserver_port (6443) +#nginx_kube_apiserver_port: 8443 + +### OTHER OPTIONAL VARIABLES +## For some things, kubelet needs to load kernel modules. For example, dynamic kernel services are needed +## for mounting persistent volumes into containers. These may not be loaded by preinstall kubernetes +## processes. For example, ceph and rbd backed volumes. Set to true to allow kubelet to load kernel +## modules. +# kubelet_load_modules: false + +## Internal network total size. This is the prefix of the +## entire network. Must be unused in your environment. +#kube_network_prefix: 18 + +## With calico it is possible to distributed routes with border routers of the datacenter. +## Warning : enabling router peering will disable calico's default behavior ('node mesh'). +## The subnets of each nodes will be distributed by the datacenter router +#peer_with_router: false + +## Upstream dns servers used by dnsmasq +upstream_dns_servers: +# - 10.10.1.241 +# - 10.10.1.242 + - 8.8.4.4 + +## There are some changes specific to the cloud providers +## for instance we need to encapsulate packets with some network plugins +## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', or 'vsphere' +## When openstack is used make sure to source in the openstack credentials +## like you would do when using nova-client before starting the playbook. +#cloud_provider: + +## When azure is used, you need to also set the following variables. +## see docs/azure.md for details on how to get these values +#azure_tenant_id: +#azure_subscription_id: +#azure_aad_client_id: +#azure_aad_client_secret: +#azure_resource_group: +#azure_location: +#azure_subnet_name: +#azure_security_group_name: +#azure_vnet_name: +#azure_route_table_name: + +## Set these proxy values in order to update docker daemon to use proxies +#http_proxy: "" +#https_proxy: "" +#no_proxy: "" + +## Uncomment this if you want to force overlay/overlay2 as docker storage driver +## Please note that overlay2 is only supported on newer kernels +#docker_storage_options: -s overlay2 + +## Default packages to install within the cluster, f.e: +#kpm_packages: +# - name: kube-system/grafana + +## Certificate Management +## This setting determines whether certs are generated via scripts or whether a +## cluster of Hashicorp's Vault is started to issue certificates (using etcd +## as a backend). Options are "script" or "vault" +cert_management: script + +## Please specify true if you want to perform a kernel upgrade +kernel_upgrade: false + diff --git a/demo/kubespray/inventory/group_vars/k8s-cluster.yml b/demo/kubespray/inventory/group_vars/k8s-cluster.yml new file mode 100644 index 0000000..d8f611a --- /dev/null +++ b/demo/kubespray/inventory/group_vars/k8s-cluster.yml @@ -0,0 +1,145 @@ +# Valid bootstrap options (required): ubuntu, coreos, centos, none +bootstrap_os: none + +#Directory where etcd data stored +etcd_data_dir: /var/lib/etcd + +# Directory where the binaries will be installed +bin_dir: /usr/local/bin + +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernets. +# This puts them in a sane location and namespace. +# Editting those values will almost surely break something. +kube_config_dir: /etc/kubernetes +kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" +kube_manifest_dir: "{{ kube_config_dir }}/manifests" +system_namespace: kube-system + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +# This is where all the cert scripts and certs will be located +kube_cert_dir: "{{ kube_config_dir }}/ssl" + +# This is where all of the bearer tokens will be stored +kube_token_dir: "{{ kube_config_dir }}/tokens" + +# This is where to save basic auth file +kube_users_dir: "{{ kube_config_dir }}/users" + +kube_api_anonymous_auth: false + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.7.3 + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" +# Random shifts for retrying failed ops like pushing/downloading +retry_stagger: 5 + +# This is the group that the cert creation scripts chgrp the +# cert files to. Not really changable... +kube_cert_group: kube-cert + +# Cluster Loglevel configuration +kube_log_level: 2 + +# Users to create for basic auth in Kubernetes API via HTTP +kube_api_pwd: "changeme" +kube_users: + kube: + pass: "{{kube_api_pwd}}" + role: admin + root: + pass: "{{kube_api_pwd}}" + role: admin + + + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +#kube_oidc_auth: false +#kube_basic_auth: false +#kube_token_auth: false + + +## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ +## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) + +# kube_oidc_url: https:// ... +# kube_oidc_client_id: kubernetes +## Optional settings for OIDC +# kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem +# kube_oidc_username_claim: sub +# kube_oidc_groups_claim: groups + + +# Choose network plugin (calico, weave or flannel) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: calico + +# Enable kubernetes network policies +enable_network_policy: false + +# Kubernetes internal network for services, unused block of space. +kube_service_addresses: 10.233.0.0/18 + +# internal network. When used, it will assign IP +# addresses from this range to individual pods. +# This network must be unused in your network infrastructure! +kube_pods_subnet: 10.233.64.0/18 + +# internal network node size allocation (optional). This is the size allocated +# to each node on your network. With these defaults you should have +# room for 4096 nodes with 254 pods per node. +kube_network_node_prefix: 24 + +# The port the API Server will be listening on. +kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}" +kube_apiserver_port: 6443 # (https) +kube_apiserver_insecure_port: 8080 # (http) + +# DNS configuration. +# Kubernetes cluster name, also will be used as DNS domain +cluster_name: k8s.corp.quobyte.com +# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods +ndots: 2 +# Can be dnsmasq_kubedns, kubedns or none +dns_mode: dnsmasq_kubedns +# Can be docker_dns, host_resolvconf or none +resolvconf_mode: docker_dns +# Deploy netchecker app to verify DNS resolve as an HTTP service +deploy_netchecker: false +# Ip address of the kubernetes skydns service +skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" +dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" +dns_domain: "{{ cluster_name }}" + +# Path used to store Docker data +docker_daemon_graph: "/var/lib/docker" + +## A string of extra options to pass to the docker daemon. +## This string should be exactly as you wish it to appear. +## An obvious use case is allowing insecure-registry access +## to self hosted registries like so: +docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }}" +docker_bin_dir: "/usr/bin" + +# Settings for containerized control plane (etcd/kubelet/secrets) +etcd_deployment_type: docker +kubelet_deployment_type: docker +cert_management: script +vault_deployment_type: docker + +# K8s image pull policy (imagePullPolicy) +k8s_image_pull_policy: IfNotPresent + +# Monitoring apps for k8s +#efk_enabled: false +efk_enabled: false + +# Helm deployment +helm_enabled: false + diff --git a/demo/kubespray/inventory/vagrant b/demo/kubespray/inventory/vagrant new file mode 100644 index 0000000..8e43184 --- /dev/null +++ b/demo/kubespray/inventory/vagrant @@ -0,0 +1,25 @@ +qb1 ansible_host=127.0.0.1 ansible_user=vagrant ansible_port=2222 access_ip=10.10.0.11 ip=10.10.0.11 ansible_ssh_private_key_file=../vagrant/.vagrant/machines/qb1/virtualbox/private_key ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +qb2 ansible_host=127.0.0.1 ansible_user=vagrant ansible_port=2200 access_ip=10.10.0.12 ip=10.10.0.12 ansible_ssh_private_key_file=../vagrant/.vagrant/machines/qb2/virtualbox/private_key ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +qb3 ansible_host=127.0.0.1 ansible_user=vagrant ansible_port=2201 access_ip=10.10.0.13 ip=10.10.0.13 ansible_ssh_private_key_file=../vagrant/.vagrant/machines/qb3/virtualbox/private_key ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +qb4 ansible_host=127.0.0.1 ansible_user=vagrant ansible_port=2202 access_ip=10.10.0.14 ip=10.10.0.14 ansible_ssh_private_key_file=../vagrant/.vagrant/machines/qb4/virtualbox/private_key ansible_ssh_common_args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + +[kube-master] +qb1 +qb2 +qb3 + +[etcd] +qb1 +qb2 +qb3 + +[kube-node] +qb1 +qb2 +qb3 +qb4 + +[k8s-cluster:children] +kube-node +kube-master + diff --git a/demo/vagrant/.gitignore b/demo/vagrant/.gitignore new file mode 100644 index 0000000..6fc3450 --- /dev/null +++ b/demo/vagrant/.gitignore @@ -0,0 +1,3 @@ +disks/ +.vagrant + diff --git a/demo/vagrant/Vagrantfile b/demo/vagrant/Vagrantfile new file mode 100644 index 0000000..bc422cc --- /dev/null +++ b/demo/vagrant/Vagrantfile @@ -0,0 +1,111 @@ +Vagrant.configure("2") do |config| + config.vm.box_check_update = false + + config.vm.define "qb1" do |qb1| + qb1.vm.box = "centos/7" + qb1.vm.hostname = "qb1" + qb1.vm.network "private_network", ip: "10.10.0.11", + virtualbox__intnet: "qbinternal" + + # use kubectl from your host + qb1.vm.network "forwarded_port", guest: 6443, host: 6443 + + qb1.vm.provider "virtualbox" do |vb| + vb.memory = 4096 + vb.cpus = 4 + unless File.exist?('./disks/qb1_data1.vdi') + # sort of a hack. if the disk file does not exist, also add the sata controller. + vb.customize ['storagectl', :id, '--name', 'SATA Controller', '--add', 'sata', '--portcount', 4] + vb.customize ['createhd', '--filename', './disks/qb1_data1.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + unless File.exist?('./disks/qb1_data2.vdi') + vb.customize ['createhd', '--filename', './disks/qb1_data2.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + unless File.exist?('./disks/qb1_data3.vdi') + vb.customize ['createhd', '--filename', './disks/qb1_data3.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', './disks/qb1_data1.vdi'] + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 2, '--device', 0, '--type', 'hdd', '--medium', './disks/qb1_data2.vdi'] + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 3, '--device', 0, '--type', 'hdd', '--medium', './disks/qb1_data3.vdi'] + end + qb1.vm.provision :shell, path: "bootstrap.sh" + end + + config.vm.define "qb2" do |qb2| + qb2.vm.box = "centos/7" + qb2.vm.hostname = "qb2" + qb2.vm.network "private_network", ip: "10.10.0.12", + virtualbox__intnet: "qbinternal" + + qb2.vm.provider "virtualbox" do |vb| + vb.memory = 2048 + vb.cpus = 4 + unless File.exist?('./disks/qb2_data1.vdi') + vb.customize ['storagectl', :id, '--name', 'SATA Controller', '--add', 'sata', '--portcount', 4] + vb.customize ['createhd', '--filename', './disks/qb2_data1.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + unless File.exist?('./disks/qb2_data2.vdi') + vb.customize ['createhd', '--filename', './disks/qb2_data2.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + unless File.exist?('./disks/qb2_data3.vdi') + vb.customize ['createhd', '--filename', './disks/qb2_data3.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', './disks/qb2_data1.vdi'] + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 2, '--device', 0, '--type', 'hdd', '--medium', './disks/qb2_data2.vdi'] + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 3, '--device', 0, '--type', 'hdd', '--medium', './disks/qb2_data3.vdi'] + end + qb2.vm.provision :shell, path: "bootstrap.sh" + end + + config.vm.define "qb3" do |qb3| + qb3.vm.box = "centos/7" + qb3.vm.hostname = "qb3" + qb3.vm.network "private_network", ip: "10.10.0.13", + virtualbox__intnet: "qbinternal" + + qb3.vm.provider "virtualbox" do |vb| + vb.memory = 2048 + vb.cpus = 4 + unless File.exist?('./disks/qb3_data1.vdi') + vb.customize ['createhd', '--filename', './disks/qb3_data1.vdi', '--variant', 'Standard', '--size', 20 * 1024] + vb.customize ['storagectl', :id, '--name', 'SATA Controller', '--add', 'sata', '--portcount', 4] + end + unless File.exist?('./disks/qb3_data2.vdi') + vb.customize ['createhd', '--filename', './disks/qb3_data2.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + unless File.exist?('./disks/qb3_data3.vdi') + vb.customize ['createhd', '--filename', './disks/qb3_data3.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', './disks/qb3_data1.vdi'] + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 2, '--device', 0, '--type', 'hdd', '--medium', './disks/qb3_data2.vdi'] + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 3, '--device', 0, '--type', 'hdd', '--medium', './disks/qb3_data3.vdi'] + end + qb3.vm.provision :shell, path: "bootstrap.sh" + end + + config.vm.define "qb4" do |qb4| + qb4.vm.box = "centos/7" + qb4.vm.hostname = "qb4" + qb4.vm.network "private_network", ip: "10.10.0.14", + virtualbox__intnet: "qbinternal" + + qb4.vm.provider "virtualbox" do |vb| + vb.memory = 2048 + vb.cpus = 4 + unless File.exist?('./disks/qb4_data1.vdi') + vb.customize ['storagectl', :id, '--name', 'SATA Controller', '--add', 'sata', '--portcount', 4] + vb.customize ['createhd', '--filename', './disks/qb4_data1.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + unless File.exist?('./disks/qb4_data2.vdi') + vb.customize ['createhd', '--filename', './disks/qb4_data2.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + unless File.exist?('./disks/qb4_data3.vdi') + vb.customize ['createhd', '--filename', './disks/qb4_data3.vdi', '--variant', 'Standard', '--size', 20 * 1024] + end + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', './disks/qb4_data1.vdi'] + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 2, '--device', 0, '--type', 'hdd', '--medium', './disks/qb4_data2.vdi'] + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 3, '--device', 0, '--type', 'hdd', '--medium', './disks/qb4_data3.vdi'] + end + qb4.vm.provision :shell, path: "bootstrap.sh" + end +end diff --git a/demo/vagrant/bootstrap.sh b/demo/vagrant/bootstrap.sh new file mode 100644 index 0000000..74d15f7 --- /dev/null +++ b/demo/vagrant/bootstrap.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +echo "10.10.0.11 qb1" >> /etc/hosts +echo "10.10.0.12 qb2" >> /etc/hosts +echo "10.10.0.13 qb3" >> /etc/hosts +echo "10.10.0.14 qb4" >> /etc/hosts + +yum install -yy epel-release +yum update +yum install ntp smartmontools + +echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf +sysctl -p /etc/sysctl.conf + diff --git a/demo/vagrant/config.txt b/demo/vagrant/config.txt new file mode 100644 index 0000000..8e21c66 --- /dev/null +++ b/demo/vagrant/config.txt @@ -0,0 +1,44 @@ +Host qb1 + HostName 127.0.0.1 + User vagrant + Port 2222 + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + PasswordAuthentication no + IdentityFile /home/meatz/Desktop/quobernetes/git/examples/vagrant/.vagrant/machines/qb1/virtualbox/private_key + IdentitiesOnly yes + LogLevel FATAL + +Host qb2 + HostName 127.0.0.1 + User vagrant + Port 2200 + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + PasswordAuthentication no + IdentityFile /home/meatz/Desktop/quobernetes/git/examples/vagrant/.vagrant/machines/qb2/virtualbox/private_key + IdentitiesOnly yes + LogLevel FATAL + +Host qb3 + HostName 127.0.0.1 + User vagrant + Port 2201 + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + PasswordAuthentication no + IdentityFile /home/meatz/Desktop/quobernetes/git/examples/vagrant/.vagrant/machines/qb3/virtualbox/private_key + IdentitiesOnly yes + LogLevel FATAL + +Host qb4 + HostName 127.0.0.1 + User vagrant + Port 2202 + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + PasswordAuthentication no + IdentityFile /home/meatz/Desktop/quobernetes/git/examples/vagrant/.vagrant/machines/qb4/virtualbox/private_key + IdentitiesOnly yes + LogLevel FATAL + diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..9879ebe --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,379 @@ +# Getting started with Quobyte Services on Kubernetes + +This setup guide assumes that you have 4 servers available. +Quobyte stores its data on dedicated disk drives, so please make sure that +the servers have some spare, unformatted SSDs or HDDs available. + +## Deploying Quobyte on a Kubernetes Cluster + +The following steps will deploy Quobyte as a cluster-wide /var/lib/kubelet/plugins/kubernetes.io~quobyte mount on a Kubernetes (>=1.1) cluster. If you use Kubernetes (>=1.4) you can use the official Quobyte Kubernetes volume plugin. + +Quobyte services use dedicated disk drives to store data. +A typical Quobyte cluster consists of 3 registries, 3 metadata servers and at least 3 data servers. + +Since we know the servers and their connected disk drives in advance, we can layout a plan for the +Quobyte deployment. To run a Quobyte service on a node, we apply labels for 'metadata', 'data', 'registry', and 'client'. +Kubernetes will make sure that the services are scheduled on the nodes. + +## Prerequisites +### Format Quobyte Devices +Quobyte is designed to run on dedicated disk drives which are formatted with ext4 or xfs and +are initialized as a Quobyte device. + +Log in the host machines and prepare the devices. In this example, we expect +`/dev/sd{b,c,d}` to be available. + +```bash +$ ssh $host + +# prepare the target mount dirs +$ for i in `seq 1 1 3`; do mkdir -p /mnt/quobyte/data_$i; done + +# partition and format disk drives +$ for dev in sdb sdc sdd; do \ + parted /dev/${dev} mklabel gpt; \ + parted /dev/${dev} mkpart primary 2048s 100%; \ + mkfs.xfs -isize=1024 /dev/${dev}1; \ + done + +# persist to fstab. + +# retrieve the PARTUUIDs for the newly created devices +$ for dev in sdb sdc sdd; do \ + blkid -s PARTUUID -o value /dev/${dev}1; \ + done + +# for each disk, add a line to fstab +PARTUUID= /mnt/quobyte/data_$i xfs relatime,logbufs=8,logbsize=256k,swalloc,allocsize=131072k + +# for SSDs we recommend mount options: +PARTUUID= /mnt/quobyte/data_$i relatime,nodiscard +``` + +When all fstab entries are created, mount the fresh disks with `mount -a`. + +Next, we need to bootstrap a registry and setup the devices to be recognized by Quobyte. +Then, prepare the other registries, metadata, and data devices as they fit your cluster setup. +Please note that every device can host all three types in parallel, but for initial setup, +it can only be initialized for a single purpose. Our target cluster will each have /mnt/quobyte/data_1 as registry +and metadata enabled, and data_2/3 as data disks. + +To achieve this, we will setup the first machine like: +```bash +$ ssh $host +$ curl -O https://raw.githubusercontent.com/quobyte/kubernetes/master/tools/qbootstrap +$ chmod +x qbootstrap + +# create a bootstrapped registry on first disk +$ sudo ./qbootstrap /mnt/quobyte/data_1 + +# and two data devices +$ curl -O https://raw.githubusercontent.com/quobyte/kubernetes/master/tools/qmkdev +$ chmod +x qmkdev +$ sudo ./qmkdev -f -t -s $(uuidgen) DATA /mnt/quobyte/data_2 +$ sudo ./qmkdev -f -t -s $(uuidgen) DATA /mnt/quobyte/data_3 +``` + +On the other 3 machines: + +```bash +$ ssh $host +$ curl -O https://raw.githubusercontent.com/quobyte/kubernetes/master/tools/qmkdev +$ chmod +x qmkdev + +# create registry - will be used for METADATA as well +$ sudo ./qmkdev -f -s $(uuidgen) -t REGISTRY /mnt/quobyte/data_1 +$ sudo ./qmkdev -f -s $(uuidgen) -t DATA /mnt/quobyte/data_2 +$ sudo ./qmkdev -f -s $(uuidgen) -t DATA /mnt/quobyte/data_3 +``` + +Please note that the target cluster will use shared REGISTRY / METADATA devices. +Hence, we first set up all these devices as REGISTRY, and later upgrade them +to also host METADATA. + +#### MountFlags + +Ensure MountFlags are shared or not set: +```bash +$ systemctl cat docker.service | grep MountFlags=shared +``` + +If not you can set the MountFlags as shared with the following commands: +```bash +$ cat << EOF > /etc/systemd/system/docker.service.d/slave-mount-flags.conf +[Service] +MountFlags=shared +EOF + +$ systemctl daemon-reload && systemctl restart docker +``` + +#### NTP + +Ensure that ntp is running on all of your nodes hosting any Quobyte service otherwise this can lead to a non working cluster. + + +## Starting the Quobyte Components + +### Create "quobyte" Namespace + +```bash +$ cd deploy +$ kubectl create -f quobyte-ns.yaml +``` + +**Additional** + +Set the default namespace of `kubectl` to the quobyte namespace: + +```bash +$ export CONTEXT=$(kubectl config view | awk '/current-context/ {print $2}') + +$ kubectl config set-context $CONTEXT --namespace=quobyte + +$ kubectl config view | grep namespace: +namespace: quobyte +``` + +If you like to reset the to the `default` namespace just repeat the steps above and replace quobyte with `default`. + +### Create Configuration + +Create the memory configuration for the Quobyte resources (if you change this file don't forget to adjust the resource limits for the components): + +```bash +$ kubectl create -f config.yaml +``` + +### Create Services + +```bash +$ kubectl create -f quobyte-services.yaml +``` + +### Deploy Registry + +Start with labeling one node as `registry`, we use this node as bootstrap node. +If you have prepared physical disks with qbootstrap, make sure that you label that host as `registry`. +Also, only label the bootstrap node. All other nodes will be added in a second step. + +```bash +$ kubectl label nodes quobyte_registry="true" +``` + +Now start the deployment for the registry. +The DaemonSet will automatically deploy a `registry` on the nodes: + +```bash +$ kubectl create -f registry-ds.yaml +$ kubectl -n quobyte get po --watch +``` + +### Deploy the API and the Web UI + +```bash +$ kubectl create -f webconsole-deployment.yaml +``` + +We have created a Quobyte management container which holds the relevant API clients to +control and monitor the cluster. +As soon as the webconsole/api pod is running: + +```bash +$ kubectl create -f qmgmt-pod.yaml +$ kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api device list +Id Host Mode Disk Used Disk Avail Services LED Mode + 1 registry-39v1s ONLINE 4 GB 40 GB REGISTRY OFF + +$ kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api registry list +Id Host Mode + 1 registry-39v1s ONLINE +``` + +Also, you can now connect to the Webconsole. + +```bash +$ kubectl port-forward 8080 > /dev/null & +``` +and point your browser to [http://localhost:8080](http://localhost:8080) to log in with default credentials "admin" and password "quobyte". + +### Deploy Remaining Registries and Upgrade Devices + +As soon as the first registry is up and running, mark the remaining nodes which contain REGISTRY devices. +Wait until all registry devices show up online. +```bash +$ kubectl label nodes quobyte_registry="true" +$ watch "kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api device list | grep REGISTRY" +``` + +You can skip this step, if you have prepared all METADATA devices with qmkdev already. +```bash + +$ kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api device list | grep REGISTRY + 1 registry-26tj1 ONLINE 34 MB 21 GB REGISTRY OFF + 4 registry-716mq ONLINE 168 MB 21 GB REGISTRY OFF + 10 registry-fnr2x ONLINE 34 MB 21 GB REGISTRY OFF + 7 registry-fxwc2 ONLINE 168 MB 21 GB REGISTRY OFF + +# for all relevant device ids update the devices to also host METADATA: +kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api device update add-type METADATA +``` + +### Deploy the Metadata and Data Service on each Node + +Add a label to all nodes that should run the Quobyte `metadata` and/or `data` service: + +```bash +$ kubectl label nodes quobyte_metadata="true" +$ kubectl label nodes quobyte_data="true" +``` + +Now you can start the DaemonSets to run the pods on these nodes. + +```bash +$ kubectl create -f data-ds.yaml +$ kubectl create -f metadata-ds.yaml +``` + +Now all Quobyte service are up and running. Check that all prepared devices are actually online: + +```bash +$ kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api device list +Id Host Mode Disk Used Disk Avail Services LED Mode + 11 data-0rhrs ONLINE 38 MB 21 GB DATA OFF + 12 data-0rhrs ONLINE 38 MB 21 GB DATA OFF + 2 data-1kv8v ONLINE 38 MB 21 GB DATA OFF + 3 data-1kv8v ONLINE 38 MB 21 GB DATA OFF + 8 data-8lsd5 ONLINE 38 MB 21 GB DATA OFF + 9 data-8lsd5 ONLINE 38 MB 21 GB DATA OFF + 5 data-tjkqs ONLINE 38 MB 21 GB DATA OFF + 6 data-tjkqs ONLINE 38 MB 21 GB DATA OFF + 1 metadata-0wtnc ONLINE 168 MB 21 GB METADATA REGISTRY OFF + 10 metadata-jdtzk ONLINE 34 MB 21 GB METADATA REGISTRY OFF + 7 metadata-md78r ONLINE 168 MB 21 GB METADATA REGISTRY OFF + 4 metadata-qkjhx ONLINE 168 MB 21 GB METADATA REGISTRY OFF +``` + + + + + +## Clients and Volumes + +To actually access data from Quobyte, we require to have a Quobyte Client running on every host. +First, create a Quobyte volume, which hosts some test data. + +```bash +$ kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api volume create testVolume root root BASE 0777 +Success. Created new volume with volume uuid ba843ca0-40e2-4f05-ae41-2813d6262201 +``` + +Then you can mount all volumes on each node at `/var/lib/kubelet/plugins/kubernetes.io~quobyte` (default plugin directory) using the client daemonset: + +```bash +$ kubectl create -f client-ds.yaml +$ kubectl label nodes quobyte_client="true" +``` + +Log into one of the hosts with ssh and check that the volume is mounted: + +```bash +$ grep "quobyte" /proc/mounts +quobyte@10.244.4.3:7866|10.244.3.3:7866|10.244.2.4:7866/cluster on /var/lib/kubelet/plugins/kubernetes.io~quobyte type fuse (rw,nosuid,nodev,noatime,user_id=0,group_id=0,default_permissions) +``` + +## Deploy a Pod using the Cluster Storage + +We prepared a simple demo pod which uses the previously prepared testVolume to demonstrate shared file system access. +It uses the [Kubernetes Quobyte Plugin](https://kubernetes.io/docs/concepts/storage/volumes/#quobyte). + +```bash +$ kubectl create -f example-pod.yaml + +$ kubectl logs example -f +Starting with a fresh state +Tue May 10 10:01:39 UTC 2016 +Tue May 10 10:01:44 UTC 2016 +Tue May 10 10:01:49 UTC 2016 +``` + +Now delete and recreate the pod: + +```bash +$ kubectl delete pod example --grace-period 0 +$ kubectl create -f example-pod.yaml +``` +Independently from the node the pod comes up on, the pod will find its old state: + +```bash +$ kubectl logs example -f +Found old state starting at Tue May 10 10:01:39 UTC 2016 +Tue May 10 10:07:32 UTC 2016 +Tue May 10 10:07:37 UTC 2016 +Tue May 10 10:07:42 UTC 2016 +``` + +## Access Webconsole and API + +### Local port-forward + +```bash +$ kubectl port-forward 8080 > /dev/null & +``` + +Access the Webconsole at + +### Ingress + +```bash +$ kubectl create -f ingress.yaml +``` + +For more information about `ingress` have a look at the [contrib repo](https://github.com/kubernetes/contrib/tree/master/ingress/controllers) and the [docs](http://kubernetes.io/docs/user-guide/ingress) + +### NodePort + +For local development you can also use a [NodePort](http://kubernetes.io/docs/user-guide/services/#type-nodeport): + +``` +$ cat webconsole-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: webconsole +spec: + type: NodePort + ports: + - name: web80 + targetPort: 8080 + port: 80 + protocol: TCP + - name: web + port: 8080 + protocol: TCP + selector: + role: webconsole + +$ cat api-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: api +spec: + type: NodePort + ports: + - name: api80 + targetPort: 7860 + port: 80 + protocol: TCP + - name: api + port: 7860 + protocol: TCP + selector: + role: webconsole +``` + +# Acknowledgement + +- [sttts](https://github.com/sttts) +- [johscheuer](https://github.com/johscheuer) diff --git a/kubernetes/client-ds.yaml b/deploy/client-ds.yaml similarity index 85% rename from kubernetes/client-ds.yaml rename to deploy/client-ds.yaml index 9136dc9..19e60f2 100644 --- a/kubernetes/client-ds.yaml +++ b/deploy/client-ds.yaml @@ -12,11 +12,12 @@ spec: prometheus.io/port: '55000' labels: role: client - version: 1.4 + version: "1.4" spec: containers: - name: quobyte-client image: quay.io/quobyte/quobyte-client:1.4 + imagePullPolicy: Always command: - /bin/sh - -xec @@ -33,19 +34,24 @@ spec: else mkdir -p ${QUOBYTE_MOUNT_POINT} fi + + # Currently, within the nsenter, k8s dns names cannot be resolved. + ADDR=$(echo $(nslookup ${QUOBYTE_REGISTRY} | grep -A10 -m1 -e 'Name:' | grep Address | awk '{split($0,a,":"); print a[2]}' | awk '{print $1":7861"}') | tr ' ' ,) + echo "QUOBYTE_REGISTRY: ${ADDR}" + /bin/nsenter -t 1 --wd=. -m -- \ lib/ld-linux-x86-64.so.2 \ --library-path ./lib \ ./bin/mount.quobyte --hostname ${NODENAME} --allow-usermapping-in-volumename \ --http-port 55000 -f -l /dev/stdout -d ${QUOBYTE_CLIENT_LOG_LEVEL} ${OPTS} \ - ${QUOBYTE_REGISTRY}/ ${QUOBYTE_MOUNT_POINT} + ${ADDR}/ ${QUOBYTE_MOUNT_POINT} securityContext: privileged: true env: - name: QUOBYTE_CLIENT_LOG_LEVEL value: INFO - name: QUOBYTE_REGISTRY - value: registry + value: registry.quobyte - name: QUOBYTE_MOUNT_POINT value: /var/lib/kubelet/plugins/kubernetes.io~quobyte - name: NODENAME diff --git a/kubernetes/config.yaml b/deploy/config.yaml similarity index 100% rename from kubernetes/config.yaml rename to deploy/config.yaml diff --git a/kubernetes/data-ds.yaml b/deploy/data-ds.yaml similarity index 72% rename from kubernetes/data-ds.yaml rename to deploy/data-ds.yaml index a323d11..570e675 100644 --- a/kubernetes/data-ds.yaml +++ b/deploy/data-ds.yaml @@ -12,7 +12,7 @@ spec: prometheus.io/port: '7873' labels: role: data - version: 1.4 + version: "1.4" spec: containers: - name: quobyte-metadata @@ -42,31 +42,21 @@ spec: - | sed "s/.*MIN_MEM_DATA=.*/MIN_MEM_DATA=${MIN_MEM}/" -i /etc/default/quobyte sed "s/.*MAX_MEM_DATA=.*/MAX_MEM_DATA=${MAX_MEM}/" -i /etc/default/quobyte - - if [ ! "$(ls -A /devices)" ] && [ ! -f /devices/QUOBYTE_DEV_SETUP ]; then - mkdir -p /devices - cat >/devices/QUOBYTE_DEV_SETUP <> /devices/UUID - fi - - cat /devices/UUID >> /etc/quobyte/$QUOBYTE_SERVICE.cfg - exec /bin/bash -x /opt/main.sh volumeMounts: - - name: devices - mountPath: /devices + - mountPath: /dev + name: dev + - mountPath: /sys/bus + name: sysbus + - mountPath: /lib/modules + name: libmodules + - mountPath: /mnt/quobyte + name: mnt-quobyte resources: requests: memory: "512Mi" cpu: "200m" - ports: + ports: - name: rpc-tcp containerPort: 7873 protocol: TCP @@ -81,9 +71,20 @@ spec: httpGet: port: 7873 path: / + securityContext: + privileged: true nodeSelector: quobyte_data: "true" volumes: - - name: devices - hostPath: - path: /mnt/data + - name: dev + hostPath: + path: /dev + - name: sysbus + hostPath: + path: /sys/bus + - name: libmodules + hostPath: + path: /lib/modules + - name: mnt-quobyte + hostPath: + path: /mnt/quobyte diff --git a/kubernetes/example-pod.yaml b/deploy/example-pod.yaml similarity index 100% rename from kubernetes/example-pod.yaml rename to deploy/example-pod.yaml diff --git a/kubernetes/ingress.yaml b/deploy/ingress.yaml similarity index 100% rename from kubernetes/ingress.yaml rename to deploy/ingress.yaml diff --git a/kubernetes/metadata-ds.yaml b/deploy/metadata-ds.yaml similarity index 73% rename from kubernetes/metadata-ds.yaml rename to deploy/metadata-ds.yaml index c90ab9b..5f76ec4 100644 --- a/kubernetes/metadata-ds.yaml +++ b/deploy/metadata-ds.yaml @@ -12,7 +12,7 @@ spec: prometheus.io/port: '7872' labels: role: metadata - version: 1.4 + version: "1.4" spec: containers: - name: quobyte-metadata @@ -42,25 +42,16 @@ spec: - | sed "s/.*MIN_MEM_METADATA=.*/MIN_MEM_METADATA=${MIN_MEM}/" -i /etc/default/quobyte sed "s/.*MAX_MEM_METADATA=.*/MAX_MEM_METADATA=${MAX_MEM}/" -i /etc/default/quobyte - if [ ! "$(ls -A /devices)" ] && [ ! -f /devices/QUOBYTE_DEV_SETUP ]; then - mkdir -p /devices - cat >/devices/QUOBYTE_DEV_SETUP <> /devices/UUID - fi - - cat /devices/UUID >> /etc/quobyte/$QUOBYTE_SERVICE.cfg - exec /bin/bash -x /opt/main.sh volumeMounts: - - name: devices - mountPath: /devices + - mountPath: /dev + name: dev + - mountPath: /sys/bus + name: sysbus + - mountPath: /lib/modules + name: libmodules + - mountPath: /mnt/quobyte + name: mnt-quobyte resources: limits: memory: "512Mi" @@ -80,9 +71,20 @@ spec: httpGet: port: 7872 path: / + securityContext: + privileged: true nodeSelector: quobyte_metadata: "true" volumes: - - name: devices - hostPath: - path: /mnt/metadata + - name: dev + hostPath: + path: /dev + - name: sysbus + hostPath: + path: /sys/bus + - name: libmodules + hostPath: + path: /lib/modules + - name: mnt-quobyte + hostPath: + path: /mnt/quobyte diff --git a/kubernetes/qmgmt-pod.yaml b/deploy/qmgmt-pod.yaml similarity index 93% rename from kubernetes/qmgmt-pod.yaml rename to deploy/qmgmt-pod.yaml index 0cfb909..ff77166 100644 --- a/kubernetes/qmgmt-pod.yaml +++ b/deploy/qmgmt-pod.yaml @@ -3,7 +3,7 @@ kind: Pod metadata: labels: role: qmgmt-pod - version: 1.4 + version: "1.4" name: qmgmt-pod namespace: quobyte spec: diff --git a/kubernetes/quobyte-ns.yaml b/deploy/quobyte-ns.yaml similarity index 100% rename from kubernetes/quobyte-ns.yaml rename to deploy/quobyte-ns.yaml diff --git a/kubernetes/quobyte-services.yaml b/deploy/quobyte-services.yaml similarity index 100% rename from kubernetes/quobyte-services.yaml rename to deploy/quobyte-services.yaml diff --git a/kubernetes/registry-ds.yaml b/deploy/registry-ds.yaml similarity index 55% rename from kubernetes/registry-ds.yaml rename to deploy/registry-ds.yaml index e96d755..0f42463 100644 --- a/kubernetes/registry-ds.yaml +++ b/deploy/registry-ds.yaml @@ -8,7 +8,7 @@ spec: metadata: labels: role: registry - version: 1.4 + version: "1.4" spec: containers: - name: quobyte-registry @@ -47,42 +47,11 @@ spec: protocol: TCP volumeMounts: - name: devices - mountPath: /devices/dev1 - command: - - /bin/bash - - -xec - - | - sed "s/.*MIN_MEM_REGISTRY=.*/MIN_MEM_REGISTRY=${MIN_MEM}/" -i /etc/default/quobyte - sed "s/.*MAX_MEM_REGISTRY=.*/MAX_MEM_REGISTRY=${MAX_MEM}/" -i /etc/default/quobyte - # TODO if directory is not empty skip (disk is mounted) - # [ ! "$(ls -A /devices)" ] && - if [ ! -f /devices/dev1/QUOBYTE_DEV_SETUP ]; then - mkdir -p /devices/dev1 - cat > /devices/dev1/QUOBYTE_DEV_SETUP <> /devices/dev1/QUOBYTE_DEV_SETUP - fi - fi - - if [ ! -f /devices/dev1/UUID ]; then - echo uuid=$(uuidgen) >> /devices/dev1/UUID - fi - - cat /devices/dev1/UUID >> /etc/quobyte/$QUOBYTE_SERVICE.cfg - - exec /bin/bash -x /opt/main.sh + mountPath: /mnt/quobyte/ resources: limits: memory: "512Mi" cpu: "100m" - lifecycle: - preStop: - exec: - command: ["/bin/bash", "-xc", "qmgmt -u api registry remove $(grep device.id= /devices/dev1/QUOBYTE_DEV_ID | cut -d= -f2)"] readinessProbe: timeoutSeconds: 5 httpGet: @@ -99,4 +68,4 @@ spec: volumes: - name: devices hostPath: - path: /mnt/registry + path: /mnt/quobyte/ diff --git a/kubernetes/webconsole-deployment.yaml b/deploy/webconsole-deployment.yaml similarity index 97% rename from kubernetes/webconsole-deployment.yaml rename to deploy/webconsole-deployment.yaml index 7f911d8..40321a8 100644 --- a/kubernetes/webconsole-deployment.yaml +++ b/deploy/webconsole-deployment.yaml @@ -8,7 +8,7 @@ spec: metadata: labels: role: webconsole - version: 1.4 + version: "1.4" spec: containers: - name: quobyte-webconsole @@ -48,7 +48,7 @@ spec: memory: "300Mi" cpu: "100m" - name: quobyte-api - image: quay.io/quobyte/quobyte-server:1.3.15 + image: quay.io/quobyte/quobyte-server:1.4 env: - name: QUOBYTE_SERVICE value: api diff --git a/kubernetes/README.md b/kubernetes/README.md deleted file mode 100644 index 4ebe71f..0000000 --- a/kubernetes/README.md +++ /dev/null @@ -1,311 +0,0 @@ -# Deploying Quobyte on a Kubernetes Cluster - -The following steps will deploy Quobyte as a cluster-wide `/var/lib/kubelet/plugins/kubernetes.io~quobyte` mount on a Kubernetes (>=1.1) cluster. If you use Kubernetes (>=1.4) you can use the official Quobyte Kubernetes volume plugin. - -## Prerequisites - -### MountFlags - -Ensure MountFlags are shared or not set: - -```bash -$ systemctl cat docker.service | grep MountFlags=shared -``` - -If not you can set the MountFlags as `shared` with the following commands: - -```bash -$ cat << EOF > /etc/systemd/system/docker.service.d/slave-mount-flags.conf -[Service] -MountFlags=shared -EOF - -$ systemctl daemon-reload && systemctl restart docker -``` - -### Fuse.conf - -Ensure that on the Host the file `/etc/fuse.conf` exists (below is an example `fuse.conf`). In the minimal Fuse config the line `user_allow_other` is needed: - -```bash -$ cat ./fuse.conf -# Allow non-root users to specify the allow_other or allow_root mount options. -user_allow_other -``` - -**Attention** - -If `/etc/fuse.conf` is missing on the host the default config from the Quobyte Clien Pod will be copied onto the host. - -### NTP - -Ensure that ntp is running on all of your nodes hosting any Quobyte service otherwise this can lead to a non working cluster. - -## Starting the Quobyte Components - -### Create "quobyte" Namespace - -```bash -$ kubectl create -f quobyte-ns.yaml -``` - -**Additional** - -Set the default namespace of `kubectl` to the quobyte namespace: - -```bash -$ export CONTEXT=$(kubectl config view | awk '/current-context/ {print $2}') - -$ kubectl config set-context $CONTEXT --namespace=quobyte - -$ kubectl config view | grep namespace: -namespace: quobyte -``` - -If you like to reset the to the `default` namespace just repeat the steps above and replace quobyte with `default`. - -### Create Configuration - -Create the memory configuration for the quobyte resources (if you change this file don't forget to adjust the resource limits for the components): - -```bash -$ kubectl create -f config.yaml -``` - -### Create Services - -```bash -$ kubectl create -f quobyte-services.yaml -``` - -### Deploy Registry - -Start with labeling one node as `registry`, we use this node as bootstrap node: - -```bash -$ kubectl label nodes quobyte_registry="true" -``` - -Now start the deployment for the registry: - -```bash -$ kubectl create -f registry-ds.yaml -``` - -Wait until the pod is up now you can label other nodes as `registry` to add additional Quobyte registries. - -```bash -$ kubectl -n quobyte get po --watch -``` - -The DaemonSet will automatically deploy a `registry` on these nodes: - -```bash -$ kubectl label nodes quobyte_registry="true" -``` - -### Deploy the API and the Web UI - -```bash -$ kubectl create -f webconsole-deployment.yaml -``` - -Wait that all devices are up, e.g. by - -```bash -$ kubectl create -f qmgmt-pod.yaml - -$ kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api:7860 device list -Id Host Mode Disk Used Disk Avail Services - 1 registry-21ndb ONLINE 4 GB 17 GB REGISTRY - 2 registry-5gtgz ONLINE 4 GB 17 GB REGISTRY - 3 registry-irvss ONLINE 4 GB 17 GB REGISTRY - -$ kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api:7860 registry list -Id Host Mode -* 1 registry-21ndb ONLINE - 2 registry-5gtgz ONLINE - 3 registry-irvss ONLINE -``` - -### Deploy the Metadata and Data Service on each Node - -Add a label to all nodes that should run the Quobyte `meta-data` and/or `data` service: - -```bash -$ kubectl label nodes quobyte_metadata="true" -$ kubectl label nodes quobyte_data="true" -``` - -Now you can start the DaemonSets to run the pods on these nodes. - -```bash -$ kubectl create -f data-ds.yaml -$ kubectl create -f metadata-ds.yaml -``` - -Wait that all devices are up with the upper `kubectl exec` command and check all services: - -```bash -$ kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api:7860 service list -Name Type UUID -data-6bdhe Data (D) 47638429-f0ed-4ea7-8f12-2452b17cde72 -data-94uc7 Data (D) c307c3fc-c7fd-45e8-a3ef-2e24ecfb23d8 -data-ogq7j Data (D) 2eb58546-a507-4a20-81b7-eccc86ff2695 -metadata-czs40 Metadata (M) c336de21-c669-4da9-90ca-dd2d9ec7977a -metadata-q3nx7 Metadata (M) f9fd3200-2c46-4dd1-8926-43be69545764 -metadata-zjz6c Metadata (M) 1c453894-aa27-4fc1-9a80-d5c54c3fde4d -registry-21ndb Registry (R) 87c78331-97d2-4878-9746-a3ff1c07c333 -registry-5gtgz Registry (R) c5c0ef49-4656-407a-845e-dc393ddcbaf4 -registry-irvss Registry (R) 6314af41-7dac-4f75-8ee9-efcebe78db26 -webconsole-3666637658-xwblz API Proxy (A) 49edfb90-cc83-47d2-9d80-1ddc45b1b9cb -webconsole-3666637658-xwblz Web Console b5e35998-0ed4-4c31-a2e0-3ff2f973851e -``` - -## Create a Volume - -Create a volume `cluster`: - -```bash -$ kubectl -n quobyte exec -it qmgmt-pod -- qmgmt -u api:7860 volume create testVolume root root BASE 0777 -``` - -Then you can mount all volumes on each node at `/var/lib/kubelet/plugins/kubernetes.io~quobyte` (default plugin directory) using the client daemonset: - -```bash -$ kubectl create -f client-ds.yaml -``` - -Log into one of the nodes with ssh and check that the volume is mounted: - -```bash -$ grep "quobyte" /proc/mounts -quobyte@10.244.4.3:7866|10.244.3.3:7866|10.244.2.4:7866/cluster on /var/lib/kubelet/plugins/kubernetes.io~quobyte type fuse (rw,nosuid,nodev,noatime,user_id=0,group_id=0,default_permissions) -``` - -## Deploy a Pod using the Cluster Storage - -### Pre 1.4 kubernetes - -```bash -$ kubectl create -f example-pod-pre.yaml - -$ kubectl logs example -f -Starting with a fresh state -Tue May 10 10:01:39 UTC 2016 -Tue May 10 10:01:44 UTC 2016 -Tue May 10 10:01:49 UTC 2016 -``` - -Now delete and recreate the pod: - -```bash -$ kubectl delete pod example --grace-period 0 - -$ kubectl create -f example-pod.yaml -``` - -Independently from the node the pod comes up on, the pod will find its old state: - -```bash -$ kubectl logs example -f -Found old state starting at Tue May 10 10:01:39 UTC 2016 -Tue May 10 10:07:32 UTC 2016 -Tue May 10 10:07:37 UTC 2016 -Tue May 10 10:07:42 UTC 2016 -``` - -### Kubernetes 1.4+ - -```bash -$ kubectl create -f example-pod.yaml - -$ kubectl logs example -f -Starting with a fresh state -Tue May 10 10:01:39 UTC 2016 -Tue May 10 10:01:44 UTC 2016 -Tue May 10 10:01:49 UTC 2016 -``` - -Now delete and recreate the pod: - -```bash -$ kubectl delete pod example --grace-period 0 - -$ kubectl create -f example-pod.yaml -``` - -Independently from the node the pod comes up on, the pod will find its old state: - -```bash -$ kubectl logs example -f -Found old state starting at Tue May 10 10:01:39 UTC 2016 -Tue May 10 10:07:32 UTC 2016 -Tue May 10 10:07:37 UTC 2016 -Tue May 10 10:07:42 UTC 2016 -``` - -## Access Webconsole and API - -### Local port-forward - -```bash -$ kubectl port-forward 8080 > /dev/null & -``` - -Access the Webconsole at - -### Ingress - -```bash -$ kubectl create -f ingress.yaml -``` - -For more information about `ingress` have a look at the [contrib repo](https://github.com/kubernetes/contrib/tree/master/ingress/controllers) and the [docs](http://kubernetes.io/docs/user-guide/ingress) - -### NodePort - -For local development you can also use a [NodePort](http://kubernetes.io/docs/user-guide/services/#type-nodeport): - -``` -$ cat webconsole-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: webconsole -spec: - type: NodePort - ports: - - name: web80 - targetPort: 8080 - port: 80 - protocol: TCP - - name: web - port: 8080 - protocol: TCP - selector: - role: webconsole - -$ cat api-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: api -spec: - type: NodePort - ports: - - name: api80 - targetPort: 7860 - port: 80 - protocol: TCP - - name: api - port: 7860 - protocol: TCP - selector: - role: webconsole -``` - -# Acknowledgement - -- [sttts](https://github.com/sttts) -- [johscheuer](https://github.com/johscheuer) diff --git a/kubernetes/example-pod-pre.yaml b/kubernetes/example-pod-pre.yaml deleted file mode 100644 index 6243912..0000000 --- a/kubernetes/example-pod-pre.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: example -spec: - containers: - - name: busybox - image: busybox:1.25 - volumeMounts: - - name: persistent - mountPath: /persistent - command: - - /bin/sh - - -ec - - | - if [ -f /persistent/log ]; then - echo -n "Found old state starting at "; head -n1 /persistent/log - else - echo -n "Starting with a fresh state" - fi - while sleep 5; do date | tee -a /persistent/log; done - volumes: - - name: persistent - hostPath: - path: /var/lib/kubelet/plugins/testVolume/example diff --git a/tools/qbootstrap b/tools/qbootstrap index 0f45ddc..0b6ef69 100755 --- a/tools/qbootstrap +++ b/tools/qbootstrap @@ -1,27 +1,39 @@ #!/bin/bash -# Copyright 2014 Quobyte Inc. +# Copyright 2017 Quobyte Inc. LANG=en_US.utf-8 DEVICE_ID_FILE="QUOBYTE_DEV_SETUP" test -z $SMARTCTL && SMARTCTL=/usr/sbin/smartctl function print_usage { - printf "Usage: %s [-f] [-y] [-s serno] mount_point\n" $(basename $0) - echo "Creates a new Quobyte USP bootstrap device on the disk/file system mounted at ." - echo "-f overwrites any existing device information (dangerous!)." - echo "-y do not ask for confirmation." - echo "-s serial_number serial number for a device." + if [[ "$(basename $0)" == qbootstrap* ]]; then + printf "Usage: %s [-f] [-t type] [-s serno] mount_point\n" $(basename $0) + echo "Creates a new Quobyte bootstrap device on the disk/file system mounted at ." + echo "-f overwrites any existing device information (dangerous!)." + echo "-s serial_number serial number for a device." + echo "-y do not ask for confirmation." + else + printf "Usage: %s [-f] [-t type] [-s serno] mount_point\n" $(basename $0) + echo "Creates a new Quobyte device on the disk/file system mounted at ." + echo "-f overwrites any existing device information (dangerous!)." + echo "-t DATA|METADATA|REGISTRY optional device type" + echo " this device should be used as." + echo "-s serial_number serial number for a device." + fi echo exit 2 } force_flag= +type_flag= serial_number= yes_mode= -while getopts fys: name +bootstrap_mode= +while getopts "fyBhs:t:" name do case $name in f) force_flag=1;; + t) type_flag="$OPTARG";; s) serial_number="$OPTARG";; y) yes_mode=1;; ?) print_usage;; @@ -29,20 +41,55 @@ do done shift $(($OPTIND - 1)) -if [ $# -eq 0 ]; then +if [[ "$(basename $0)" == qbootstrap* ]] +then + bootstrap_mode=1 +fi + +if [ $# -ne 1 ]; then print_usage fi if [ -n "$EUID" -a $EUID -ne 0 ]; then - echo "Error: This programm must be executed as root (requires access to smartctl to read device identity)." + echo "Error: This programm must be executed as root (requires access to smartctl/lshw to read device identity)." exit 4 fi -MOUNT_POINT=$(echo "$1"|sed 's/\/$//g') +MOUNT_POINT="$(readlink -f "$1")" + +if [[ $bootstrap_mode == 1 ]]; then + if [[ "$type_flag" != "" ]];then + echo "Cannot specify type when creating bootstrap registry device." + exit 11 + fi + type_flag="R" + + if [ -z "$yes_mode" ]; then + echo "Attention: a bootstrap device must only be created once per Quobyte installation." + echo "If you have already created a bootstrap device on any machine, please use qmkdev -t to create further devices." + read -p "Are you sure you that you want to make ${MOUNT_POINT} your installation's bootstrap device (y/n)? " -n 1 -r + echo # (optional) move to a new line + if [[ ! $REPLY =~ ^[Yy]$ ]] + then + echo "Cancelled." + exit 8 + fi + else + echo "Skipping confirmation." + fi +fi + echo "Device mount point: ${MOUNT_POINT}" +type_flag=$(echo "$type_flag"|awk '{print toupper($0)}') +if [[ "$type_flag" != "" && "$type_flag" != "REGISTRY" && \ + "$type_flag" != "METADATA" && "$type_flag" != "DATA" &&\ + "$type_flag" != "D" && "$type_flag" != "M" && "$type_flag" != "R" ]]; then + echo "Error: Invalid device type." + exit 10 +fi -DEVICE_FILE=${MOUNT_POINT}/${DEVICE_ID_FILE} -if [ -e ${DEVICE_FILE} ]; then +DEVICE_FILE="${MOUNT_POINT}/${DEVICE_ID_FILE}" +if [ -e "${DEVICE_FILE}" ]; then if [ -z "$force_flag" ]; then echo "Error: Selected mount point is already a Quobyte device." echo " Use -f to overwrite the current device information." @@ -54,19 +101,31 @@ if [ -e ${DEVICE_FILE} ]; then fi fi -DEVICE=`grep " ${MOUNT_POINT} " /proc/mounts | cut -d " " -f 1` -FSTYPE=`grep " ${MOUNT_POINT} " /proc/mounts | cut -d " " -f 3` + +escaped="${MOUNT_POINT// /\\040}" +# Ignore any rootfs mounts. +lines="$(fgrep " ${escaped} " /proc/mounts | fgrep -v " rootfs ")" +if [[ "$(wc -l <<<"${lines}")" -gt 1 ]]; then + echo "Error: Ambiguous mount point. Multiple matches for ' ${escaped} ' in /proc/mounts." + exit 11 +fi +if [[ "$(cut -d " " -f 2 <<<"${lines}")" != "${escaped}" ]]; then + echo "Error: Invalid mount point. ' ${escaped} ' not found in second column of /proc/mounts." + exit 11 +fi +DEVICE="$(cut -d " " -f 1 <<<"${lines}")" +FSTYPE="$(cut -d " " -f 3 <<<"${lines}")" if [ -z "${DEVICE}" ]; then - echo "Error: Invalid mount point. ' ${MOUNT_POINT} ' not found in /proc/mounts." + echo "Error: Invalid mount point. ' ${escaped} ' not found in /proc/mounts." exit 11 fi if [ ! -b "${DEVICE}" ]; then echo "Error: Invalid mount point. ${DEVICE} is no block device." exit 12 fi -# NOTE(quobyte): Keep this list in sync with the one in the OSD. -SUPPORTED_FS_TYPES="ext4 xfs btrfs" +# NOTE(quobyte): Keep this list in sync with the one in the data service. +SUPPORTED_FS_TYPES="ext4 xfs" supported_fs_found=0 for expected_fs in $SUPPORTED_FS_TYPES do @@ -82,11 +141,91 @@ fi echo "Linux device: ${DEVICE}" -DEVICE_MODEL=`$SMARTCTL --info ${DEVICE} | \ - sed -n 's/Device Model:\s*\(.*\)$/\1/p'` -if [ -z "$DEVICE_MODEL" ]; then - DEVICE_MODEL=`$SMARTCTL --info ${DEVICE} | \ - sed -n 's/Product:\s*\([-\._a-zA-Z0-9]*\).*/\1/p'` +smartctl_available() { + test -x "$(type -P "${SMARTCTL}")" +} + +set_device_model_smartctl() { + local DEVICE="$1" + DEVICE_MODEL="$("${SMARTCTL}" --info "${DEVICE}" | \ + sed -n 's/Device Model:\s*\(.*\)$/\1/p')" + if [ -z "$DEVICE_MODEL" ]; then + DEVICE_MODEL="$("${SMARTCTL}" --info "${DEVICE}" | \ + sed -n 's/Product:\s*\([-\._a-zA-Z0-9]*\).*/\1/p')" + fi +} + +set_device_serial_smartctl() { + local DEVICE="$1" + DEVICE_SERIAL="$("${SMARTCTL}" --info "${DEVICE}" | \ + sed -n 's/Serial.*:\s*\([-\._a-zA-Z0-9]*\).*/\1/p')" +} + +set_device_model_and_serial_lshw() { + local device="$1" + local lshw="$(type -P "${LSHW:-lshw}" /usr/sbin/lshw /usr/bin/lshw | head -n1)" + + # Parse XML as "lshw -short" output does not contain serial number... + local line="$("${lshw}" -quiet -class disk -xml | awk ' + BEGIN { + node_level = 0 + disk_node_entered_at = -1 + + logicalname = "" + serial = "" + product = "" + } + + /^ */ { + if (node_level == disk_node_entered_at) { + product = gensub("^ *([^<]+)", "\\1", 1) + } + next + } + /^ */ { + if (node_level == disk_node_entered_at) { + logicalname = gensub("^ *([^<]+)", "\\1", 1) + } + next + } + /^ */ { + if (node_level == disk_node_entered_at) { + serial = gensub("^ *([^<]+)", "\\1", 1) + } + next + } + + /^ *<\/node>/ { + if (node_level == disk_node_entered_at) { + disk_node_entered_at = -1 + printf("%s,%s,%s\n", logicalname, serial, product) + + logicalname = "" + serial = "" + product = "" + } + node_level -= 1 + } + ' | fgrep "${device}," | head -n1)" + + DEVICE_MODEL="$(cut -d',' -f 3 <<<"${line}")" + DEVICE_SERIAL="$(cut -d',' -f 2 <<<"${line}")" +} + + +if smartctl_available; then + set_device_model_smartctl "${DEVICE}" +else + echo "INFO: smartctl not available, falling back to lshw" + set_device_model_and_serial_lshw "${DEVICE}" fi if [ -z "$DEVICE_MODEL" ]; then DEVICE_MODEL="unknown" @@ -98,9 +237,6 @@ if [ -n "$serial_number" ]; then echo "Override Serial Number: ${serial_number}" DEVICE_SERIAL="${serial_number}" else - DEVICE_SERIAL=`$SMARTCTL --info ${DEVICE} | \ - sed -n 's/Serial.*:\s*\([-\._a-zA-Z0-9]*\).*/\1/p'` - if [ $generate_random_serial -eq 0 ]; then if [[ "$DEVICE_MODEL" =~ ^QEMU.* ]]; then echo "Ignoring pseudo serial number from QEMU device. Generating uuid." @@ -108,9 +244,12 @@ else elif [[ "$DEVICE_MODEL" =~ ^VBOX.* ]]; then echo "Ignoring pseudo serial number from VBOX device. Generating uuid." generate_random_serial=1 - elif [ -z "${DEVICE_SERIAL}" ]; then - echo "Cannot determine device serial number. Generating uuid." - generate_random_serial=1 + else + smartctl_available && set_device_serial_smartctl "${DEVICE}" + if [ -z "${DEVICE_SERIAL}" ]; then + echo "Cannot determine device serial number. Generating uuid." + generate_random_serial=1 + fi fi fi fi @@ -127,35 +266,54 @@ if [ $generate_random_serial -eq 1 ]; then fi fi -if [ -z "$yes_mode" ]; then - echo "Attention: a bootstrap device must only be created once per Quobyte installation." - echo "If you have already created a bootstrap device on any machine, please use qmkdev to create further devices." - read -p "Are you sure you that you want to make ${MOUNT_POINT} your installation's bootstrap device (y/n)? " -n 1 -r - echo # (optional) move to a new line - if [[ ! $REPLY =~ ^[Yy]$ ]] - then - echo "Cancelled." - exit 8 +if [[ "$type_flag" == "DATA" || "$type_flag" == "D" ]]; then + AVAIL_SPACE=`df -B1 $MOUNT_POINT | grep -v Avail | awk '{ print $4 "\t" }'` + AVAIL_SPACE_HR=`df -h $MOUNT_POINT | grep -v Avail | awk '{ print $4 "\t" }' | tr -d '[[:space:]]'` + if [ $AVAIL_SPACE -lt 21474836480 ]; then + echo "WARNING: $MOUNT_POINT has less than 20G of available space ($AVAIL_SPACE_HR)." + echo "Quobyte may refuse to create new files if such devices are used as data devices." + echo "We highly recommend to use a device with more capacity!" fi -else - echo "Skipping confirmation." fi - + CREATION_DATE=`date` umask 0022 -echo "# Quobyte USP bootstrap device identifier file" > ${DEVICE_FILE} -echo "# Created ${CREATION_DATE}" >> ${DEVICE_FILE} -echo "# Hostname: ${HOSTNAME}" >> ${DEVICE_FILE} -echo "device.serial=${DEVICE_SERIAL}" >> ${DEVICE_FILE} -echo "device.model=${DEVICE_MODEL}" >> ${DEVICE_FILE} -echo "device.bootstrap=true" >> ${DEVICE_FILE} -echo "device.type=DIR_DEVICE" >> ${DEVICE_FILE} - -chown quobyte ${DEVICE_FILE} -chown quobyte ${MOUNT_POINT} +echo "# Quobyte device identifier file" > "${DEVICE_FILE}" +echo "# Created ${CREATION_DATE}" >> "${DEVICE_FILE}" +echo "# Hostname: ${HOSTNAME}" >> "${DEVICE_FILE}" +echo "device.serial=${DEVICE_SERIAL}" >> "${DEVICE_FILE}" +echo "device.model=${DEVICE_MODEL}" >> "${DEVICE_FILE}" +if [ -n "$type_flag" ]; then + if [[ "$type_flag" == "METADATA" || "$type_flag" == "M" ]]; then + echo "device.type=METADATA_DEVICE" >> "${DEVICE_FILE}" + elif [[ "$type_flag" == "DATA" || "$type_flag" == "D" ]]; then + echo "device.type=DATA_DEVICE" >> "${DEVICE_FILE}" + elif [[ "$type_flag" == "REGISTRY" || "$type_flag" == "R" ]]; then + if [[ $bootstrap_mode == 1 ]]; then + echo "device.bootstrap=true" >> "${DEVICE_FILE}" + fi + echo "device.type=DIR_DEVICE" >> "${DEVICE_FILE}" + fi +else + echo "No type specified. To use the device, please assign one or more content" + echo "types using the qmgmt tool." +fi +chown quobyte "${DEVICE_FILE}" +chown quobyte "${MOUNT_POINT}" if [[ $? != 0 ]]; then echo "WARNING: Cannot change ownership of mount point, please make sure Quobyte services" echo " can write to the mount point!" fi -echo "Bootstrap device successfully initialized." +if [ "$FSTYPE" = "ext4" ]; then + tune2fs >/dev/null -m 0 "${DEVICE}" +fi + +if [[ $bootstrap_mode == 1 ]]; then + echo "Bootstrap device successfully initialized." +else + echo "Device successfully initialized." +fi +echo +echo "If this device is mounted on a host with a running Quobyte service, " +echo "the device will be registered automatically." diff --git a/tools/qmkdev b/tools/qmkdev index fc68e95..0b6ef69 100755 --- a/tools/qmkdev +++ b/tools/qmkdev @@ -1,17 +1,25 @@ #!/bin/bash -# Copyright 2014 Quobyte Inc. +# Copyright 2017 Quobyte Inc. LANG=en_US.utf-8 DEVICE_ID_FILE="QUOBYTE_DEV_SETUP" test -z $SMARTCTL && SMARTCTL=/usr/sbin/smartctl function print_usage { - printf "Usage: %s [-f] [-t type] [-s serno] mount_point\n" $(basename $0) - echo "Creates a new Quobyte device on the disk/file system mounted at ." - echo "-f overwrites any existing device information (dangerous!)." - echo "-t DATA|METADATA|REGISTRY optional device type" - echo " this device should be used as." - echo "-s serial_number serial number for a device." + if [[ "$(basename $0)" == qbootstrap* ]]; then + printf "Usage: %s [-f] [-t type] [-s serno] mount_point\n" $(basename $0) + echo "Creates a new Quobyte bootstrap device on the disk/file system mounted at ." + echo "-f overwrites any existing device information (dangerous!)." + echo "-s serial_number serial number for a device." + echo "-y do not ask for confirmation." + else + printf "Usage: %s [-f] [-t type] [-s serno] mount_point\n" $(basename $0) + echo "Creates a new Quobyte device on the disk/file system mounted at ." + echo "-f overwrites any existing device information (dangerous!)." + echo "-t DATA|METADATA|REGISTRY optional device type" + echo " this device should be used as." + echo "-s serial_number serial number for a device." + fi echo exit 2 } @@ -19,39 +27,69 @@ function print_usage { force_flag= type_flag= serial_number= -while getopts fhs:t: name +yes_mode= +bootstrap_mode= +while getopts "fyBhs:t:" name do case $name in f) force_flag=1;; t) type_flag="$OPTARG";; s) serial_number="$OPTARG";; + y) yes_mode=1;; ?) print_usage;; esac done shift $(($OPTIND - 1)) -type_flag=$(echo "$type_flag"|awk '{print toupper($0)}') -if [[ "$type_flag" != "" && "$type_flag" != "REGISTRY" && \ - "$type_flag" != "METADATA" && "$type_flag" != "DATA" &&\ - "$type_flag" != "D" && "$type_flag" != "M" && "$type_flag" != "R" ]]; then - echo "Error: Invalid device type." - exit 10 +if [[ "$(basename $0)" == qbootstrap* ]] +then + bootstrap_mode=1 fi -if [ $# -eq 0 ]; then +if [ $# -ne 1 ]; then print_usage fi if [ -n "$EUID" -a $EUID -ne 0 ]; then - echo "Error: This programm must be executed as root (requires access to smartctl to read device identity)." + echo "Error: This programm must be executed as root (requires access to smartctl/lshw to read device identity)." exit 4 fi -MOUNT_POINT=$(echo "$1"|sed 's/\/$//g') +MOUNT_POINT="$(readlink -f "$1")" + +if [[ $bootstrap_mode == 1 ]]; then + if [[ "$type_flag" != "" ]];then + echo "Cannot specify type when creating bootstrap registry device." + exit 11 + fi + type_flag="R" + + if [ -z "$yes_mode" ]; then + echo "Attention: a bootstrap device must only be created once per Quobyte installation." + echo "If you have already created a bootstrap device on any machine, please use qmkdev -t to create further devices." + read -p "Are you sure you that you want to make ${MOUNT_POINT} your installation's bootstrap device (y/n)? " -n 1 -r + echo # (optional) move to a new line + if [[ ! $REPLY =~ ^[Yy]$ ]] + then + echo "Cancelled." + exit 8 + fi + else + echo "Skipping confirmation." + fi +fi + echo "Device mount point: ${MOUNT_POINT}" +type_flag=$(echo "$type_flag"|awk '{print toupper($0)}') +if [[ "$type_flag" != "" && "$type_flag" != "REGISTRY" && \ + "$type_flag" != "METADATA" && "$type_flag" != "DATA" &&\ + "$type_flag" != "D" && "$type_flag" != "M" && "$type_flag" != "R" ]]; then + echo "Error: Invalid device type." + exit 10 +fi -DEVICE_FILE=${MOUNT_POINT}/${DEVICE_ID_FILE} -if [ -e ${DEVICE_FILE} ]; then +DEVICE_FILE="${MOUNT_POINT}/${DEVICE_ID_FILE}" +if [ -e "${DEVICE_FILE}" ]; then if [ -z "$force_flag" ]; then echo "Error: Selected mount point is already a Quobyte device." echo " Use -f to overwrite the current device information." @@ -63,19 +101,31 @@ if [ -e ${DEVICE_FILE} ]; then fi fi -DEVICE=`grep " ${MOUNT_POINT} " /proc/mounts | cut -d " " -f 1` -FSTYPE=`grep " ${MOUNT_POINT} " /proc/mounts | cut -d " " -f 3` + +escaped="${MOUNT_POINT// /\\040}" +# Ignore any rootfs mounts. +lines="$(fgrep " ${escaped} " /proc/mounts | fgrep -v " rootfs ")" +if [[ "$(wc -l <<<"${lines}")" -gt 1 ]]; then + echo "Error: Ambiguous mount point. Multiple matches for ' ${escaped} ' in /proc/mounts." + exit 11 +fi +if [[ "$(cut -d " " -f 2 <<<"${lines}")" != "${escaped}" ]]; then + echo "Error: Invalid mount point. ' ${escaped} ' not found in second column of /proc/mounts." + exit 11 +fi +DEVICE="$(cut -d " " -f 1 <<<"${lines}")" +FSTYPE="$(cut -d " " -f 3 <<<"${lines}")" if [ -z "${DEVICE}" ]; then - echo "Error: Invalid mount point. ' ${MOUNT_POINT} ' not found in /proc/mounts." + echo "Error: Invalid mount point. ' ${escaped} ' not found in /proc/mounts." exit 11 fi if [ ! -b "${DEVICE}" ]; then echo "Error: Invalid mount point. ${DEVICE} is no block device." exit 12 fi -# NOTE(quobyte): Keep this list in sync with the one in the OSD. -SUPPORTED_FS_TYPES="ext4 xfs btrfs" +# NOTE(quobyte): Keep this list in sync with the one in the data service. +SUPPORTED_FS_TYPES="ext4 xfs" supported_fs_found=0 for expected_fs in $SUPPORTED_FS_TYPES do @@ -91,11 +141,91 @@ fi echo "Linux device: ${DEVICE}" -DEVICE_MODEL=`$SMARTCTL --info ${DEVICE} | \ - sed -n 's/Device Model:\s*\(.*\)$/\1/p'` -if [ -z "$DEVICE_MODEL" ]; then - DEVICE_MODEL=`$SMARTCTL --info ${DEVICE} | \ - sed -n 's/Product:\s*\([-\._a-zA-Z0-9]*\).*/\1/p'` +smartctl_available() { + test -x "$(type -P "${SMARTCTL}")" +} + +set_device_model_smartctl() { + local DEVICE="$1" + DEVICE_MODEL="$("${SMARTCTL}" --info "${DEVICE}" | \ + sed -n 's/Device Model:\s*\(.*\)$/\1/p')" + if [ -z "$DEVICE_MODEL" ]; then + DEVICE_MODEL="$("${SMARTCTL}" --info "${DEVICE}" | \ + sed -n 's/Product:\s*\([-\._a-zA-Z0-9]*\).*/\1/p')" + fi +} + +set_device_serial_smartctl() { + local DEVICE="$1" + DEVICE_SERIAL="$("${SMARTCTL}" --info "${DEVICE}" | \ + sed -n 's/Serial.*:\s*\([-\._a-zA-Z0-9]*\).*/\1/p')" +} + +set_device_model_and_serial_lshw() { + local device="$1" + local lshw="$(type -P "${LSHW:-lshw}" /usr/sbin/lshw /usr/bin/lshw | head -n1)" + + # Parse XML as "lshw -short" output does not contain serial number... + local line="$("${lshw}" -quiet -class disk -xml | awk ' + BEGIN { + node_level = 0 + disk_node_entered_at = -1 + + logicalname = "" + serial = "" + product = "" + } + + /^ */ { + if (node_level == disk_node_entered_at) { + product = gensub("^ *([^<]+)", "\\1", 1) + } + next + } + /^ */ { + if (node_level == disk_node_entered_at) { + logicalname = gensub("^ *([^<]+)", "\\1", 1) + } + next + } + /^ */ { + if (node_level == disk_node_entered_at) { + serial = gensub("^ *([^<]+)", "\\1", 1) + } + next + } + + /^ *<\/node>/ { + if (node_level == disk_node_entered_at) { + disk_node_entered_at = -1 + printf("%s,%s,%s\n", logicalname, serial, product) + + logicalname = "" + serial = "" + product = "" + } + node_level -= 1 + } + ' | fgrep "${device}," | head -n1)" + + DEVICE_MODEL="$(cut -d',' -f 3 <<<"${line}")" + DEVICE_SERIAL="$(cut -d',' -f 2 <<<"${line}")" +} + + +if smartctl_available; then + set_device_model_smartctl "${DEVICE}" +else + echo "INFO: smartctl not available, falling back to lshw" + set_device_model_and_serial_lshw "${DEVICE}" fi if [ -z "$DEVICE_MODEL" ]; then DEVICE_MODEL="unknown" @@ -107,9 +237,6 @@ if [ -n "$serial_number" ]; then echo "Override Serial Number: ${serial_number}" DEVICE_SERIAL="${serial_number}" else - DEVICE_SERIAL=`$SMARTCTL --info ${DEVICE} | \ - sed -n 's/Serial.*:\s*\([-\._a-zA-Z0-9]*\).*/\1/p'` - if [ $generate_random_serial -eq 0 ]; then if [[ "$DEVICE_MODEL" =~ ^QEMU.* ]]; then echo "Ignoring pseudo serial number from QEMU device. Generating uuid." @@ -117,9 +244,12 @@ else elif [[ "$DEVICE_MODEL" =~ ^VBOX.* ]]; then echo "Ignoring pseudo serial number from VBOX device. Generating uuid." generate_random_serial=1 - elif [ -z "${DEVICE_SERIAL}" ]; then - echo "Cannot determine device serial number. Generating uuid." - generate_random_serial=1 + else + smartctl_available && set_device_serial_smartctl "${DEVICE}" + if [ -z "${DEVICE_SERIAL}" ]; then + echo "Cannot determine device serial number. Generating uuid." + generate_random_serial=1 + fi fi fi fi @@ -136,33 +266,54 @@ if [ $generate_random_serial -eq 1 ]; then fi fi +if [[ "$type_flag" == "DATA" || "$type_flag" == "D" ]]; then + AVAIL_SPACE=`df -B1 $MOUNT_POINT | grep -v Avail | awk '{ print $4 "\t" }'` + AVAIL_SPACE_HR=`df -h $MOUNT_POINT | grep -v Avail | awk '{ print $4 "\t" }' | tr -d '[[:space:]]'` + if [ $AVAIL_SPACE -lt 21474836480 ]; then + echo "WARNING: $MOUNT_POINT has less than 20G of available space ($AVAIL_SPACE_HR)." + echo "Quobyte may refuse to create new files if such devices are used as data devices." + echo "We highly recommend to use a device with more capacity!" + fi +fi + CREATION_DATE=`date` umask 0022 -echo "# Quobyte device identifier file" > ${DEVICE_FILE} -echo "# Created ${CREATION_DATE}" >> ${DEVICE_FILE} -echo "# Hostname: ${HOSTNAME}" >> ${DEVICE_FILE} -echo "device.serial=${DEVICE_SERIAL}" >> ${DEVICE_FILE} -echo "device.model=${DEVICE_MODEL}" >> ${DEVICE_FILE} +echo "# Quobyte device identifier file" > "${DEVICE_FILE}" +echo "# Created ${CREATION_DATE}" >> "${DEVICE_FILE}" +echo "# Hostname: ${HOSTNAME}" >> "${DEVICE_FILE}" +echo "device.serial=${DEVICE_SERIAL}" >> "${DEVICE_FILE}" +echo "device.model=${DEVICE_MODEL}" >> "${DEVICE_FILE}" if [ -n "$type_flag" ]; then if [[ "$type_flag" == "METADATA" || "$type_flag" == "M" ]]; then - echo "device.type=METADATA_DEVICE" >> ${DEVICE_FILE} + echo "device.type=METADATA_DEVICE" >> "${DEVICE_FILE}" elif [[ "$type_flag" == "DATA" || "$type_flag" == "D" ]]; then - echo "device.type=DATA_DEVICE" >> ${DEVICE_FILE} + echo "device.type=DATA_DEVICE" >> "${DEVICE_FILE}" elif [[ "$type_flag" == "REGISTRY" || "$type_flag" == "R" ]]; then - echo "device.type=DIR_DEVICE" >> ${DEVICE_FILE} + if [[ $bootstrap_mode == 1 ]]; then + echo "device.bootstrap=true" >> "${DEVICE_FILE}" + fi + echo "device.type=DIR_DEVICE" >> "${DEVICE_FILE}" fi else echo "No type specified. To use the device, please assign one or more content" echo "types using the qmgmt tool." fi -chown quobyte ${DEVICE_FILE} -chown quobyte ${MOUNT_POINT} +chown quobyte "${DEVICE_FILE}" +chown quobyte "${MOUNT_POINT}" if [[ $? != 0 ]]; then echo "WARNING: Cannot change ownership of mount point, please make sure Quobyte services" echo " can write to the mount point!" fi -echo "Device successfully initialized." +if [ "$FSTYPE" = "ext4" ]; then + tune2fs >/dev/null -m 0 "${DEVICE}" +fi + +if [[ $bootstrap_mode == 1 ]]; then + echo "Bootstrap device successfully initialized." +else + echo "Device successfully initialized." +fi echo echo "If this device is mounted on a host with a running Quobyte service, " echo "the device will be registered automatically."