Large tests and runnable example

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "scripts/test/pytest"]
+	path = scripts/test/pytest
+	url = https://github.com/marcin-krolik/snap-pytest

examples/tasks/README.md

@@ -0,0 +1,27 @@
+# Example tasks
+
+[This](task-load.json) example task will collect metrics from **load** and publish
+them to file.  
+
+## Running the example
+
+### Requirements 
+ * `docker` and `docker-compose` are **installed** and **configured** 
+
+Running the sample is as *easy* as running the script `./run-mock-load.sh`.
+
+## Files
+- [mock-load.sh](mock-load.sh)
+    - Downloads `snapd`, `snapctl`, `snap-plugin-collector-load`,
+        `snap-plugin-publisher-mock-file` and starts the task
+- [run-mock-load.sh](run-mock-load.sh)
+    - The example is launched with this script     
+- [task-load.json](task-load.json)
+    - Snap task definition
+- [.setup.sh](.setup.sh)
+    - Verifies dependencies and starts the containers.  It's called 
+    by [run-mock-load.sh](run-mock-load.sh).
+- [docker-compose.yml](docker-compose.yml)
+    - A docker compose file which defines "runner" container where snapd
+     is run from. You will be dumped into a shell in this container
+     after running.    

examples/tasks/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '2'
+services:
+  main:
+    container_name: runner
+    image: intelsdi/snap:alpine
+    environment:
+     - SNAP_VERSION=latest
+    volumes:
+      - ${PLUGIN_SRC}:/snap-plugin-collector-load
+      - /proc:/var/procfs
+    network_mode: "host"

examples/tasks/mock-load.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+# get the directory the script exists in
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# source the common bash script 
+. "${__dir}/../../scripts/common.sh"
+
+# ensure PLUGIN_PATH is set
+TMPDIR=${TMPDIR:-"/tmp"}
+PLUGIN_PATH=${PLUGIN_PATH:-"${TMPDIR}/snap/plugins"}
+mkdir -p $PLUGIN_PATH
+
+_info "downloading plugins"
+(cd $PLUGIN_PATH && curl -sSO http://snap.ci.snap-telemetry.io/snap/master/latest/snap-plugin-publisher-mock-file && chmod 755 snap-plugin-publisher-mock-file)
+(cd $PLUGIN_PATH && curl -sSO http://snap.ci.snap-telemetry.io/snap/master/latest/snap-plugin-processor-passthru && chmod 755 snap-plugin-processor-passthru)
+(cd $PLUGIN_PATH && curl -sSO http://snap.ci.snap-telemetry.io/plugins/snap-plugin-collector-load/latest/linux/x86_64/snap-plugin-collector-load && chmod 755 snap-plugin-collector-load)
+
+SNAP_FLAG=0
+
+# this block will wait check if snapctl and snapd are loaded before the plugins are loaded and the task is started
+ for i in `seq 1 10`; do
+            _info "try ${i}"
+             if [[ -f /usr/local/bin/snapctl && -f /usr/local/bin/snapd ]];
+                then
+                    _info "loading plugins"
+                    snapctl plugin load "${PLUGIN_PATH}/snap-plugin-publisher-mock-file"
+                    snapctl plugin load "${PLUGIN_PATH}/snap-plugin-processor-passthru"
+                    snapctl plugin load "${PLUGIN_PATH}/snap-plugin-collector-load"
+
+                    _info "creating and starting a task"
+                    snapctl task create -t "${__dir}/task-load.json"
+
+                    SNAP_FLAG=1
+
+                    break
+             fi 
+
+        _info "snapctl and/or snapd are unavailable, sleeping for 5 seconds"
+        sleep 5
+done 
+
+
+# check if snapctl/snapd have loaded
+if [ $SNAP_FLAG -eq 0 ]
+    then
+     echo "Could not load snapctl or snapd"
+     exit 1
+fi

examples/tasks/run-mock-load.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+# get the directory the script exists in
+__dir=$(cd $(dirname ${BASH_SOURCE[0]})/../../scripts/docker/large && pwd)
+__proj_dir="$(cd $(dirname ${BASH_SOURCE[0]})/../../ && pwd)"
+__proj_name="$(basename $__proj_dir)"
+
+export PLUGIN_SRC="${__proj_dir}"
+
+# verifies dependencies
+. "${__proj_dir}/examples/tasks/.setup.sh"
+
+# downloads plugins, starts snap, load plugins and start a task
+cd "${__proj_dir}/examples/tasks" && docker-compose exec main bash -c "PLUGIN_PATH=/etc/snap/plugins /${__proj_name}/examples/tasks/mock-load.sh && printf \"\n\nhint: type 'snapctl task list'\ntype 'exit' when your done\n\n\" && bash"

examples/task/load-file.json → examples/tasks/task-load.json

@@ -9,11 +9,11 @@
             "metrics": {
                 "/intel/procfs/load/min1": {},
                 "/intel/procfs/load/min15": {},
-                "/intel/procfs/load/scheduling": {}
+                "/intel/procfs/load/runnable_scheduling": {}
             },
             "config": {
-                "/intel/procfs/load/": {
-                    "proc_path": "/var/procfs/loadavg"
+                "/intel/procfs/load": {
+                    "proc_path": "/var/procfs"
                 }
             },
             "process": [
@@ -22,7 +22,7 @@
                     "process": null,
                     "publish": [
                         {
-                            "plugin_name": "file",
+                            "plugin_name": "mock-file",
                             "config": {
                                 "file": "/tmp/published_load"
                             }

scripts/config/k8s-deployment.yml

@@ -0,0 +1,26 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata: 
+    name: load-deployment
+spec:
+    replicas: 1
+    template:
+        metadata:
+          labels:
+            app: load-large-test
+        spec:
+          containers:
+            - name: load-large-test
+              image: mkrolik/snap-pytest:lscpu
+              imagePullPolicy: "IfNotPresent"
+              volumeMounts:
+              - mountPath: /var/procfs
+                name: procfs
+              command:
+                - sleep
+                - "3000"
+          volumes:
+            - name: procfs
+              hostPath:
+                path: /proc

scripts/docker/large/docker-compose.yml

@@ -0,0 +1,10 @@
+version: '2'
+services:
+  main:
+    container_name: load_large_test
+    image: mkrolik/python-apline:lscpu
+    network_mode: "host"
+    volumes:
+      - ${PLUGIN_SRC}:/snap-plugin-collector-load
+      - /proc:/var/procfs
+    entrypoint: sh -c 'python /snap-plugin-collector-load/scripts/test/large.py'

scripts/large_compose.sh

@@ -0,0 +1,56 @@
+#!/bin/bash 
+
+#http://www.apache.org/licenses/LICENSE-2.0.txt
+#
+#
+#Copyright 2016 Intel Corporation
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+set -e
+set -u
+set -o pipefail
+
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+__proj_dir="$(dirname "$__dir")"
+__proj_name="$(basename $__proj_dir)"
+
+. "${__dir}/common.sh"
+
+# NOTE: these variables control the docker-compose image.
+export PLUGIN_SRC="${__proj_dir}"
+export LOG_LEVEL="${LOG_LEVEL:-"7"}"
+export PROJECT_NAME="${__proj_name}"
+
+TEST_TYPE="${TEST_TYPE:-"large"}"
+
+docker_folder="${__proj_dir}/scripts/docker/${TEST_TYPE}"
+
+_docker_project () {
+  echo ${docker_folder}
+  cd "${docker_folder}" && "$@"
+}
+
+_debug "updating repository submodule with pytest"
+git submodule update --init --recursive
+
+_debug "building docker compose images"
+_docker_project docker-compose build
+_debug "running test: ${TEST_TYPE}"
+_docker_project docker-compose up
+test_res=`docker-compose ps -q | xargs docker inspect -f '{{ .Name }} exited with status {{ .State.ExitCode }}' | awk '{print $5}'`
+echo "exit code from large_jenkins $test_res"
+_debug "stopping docker compose images"
+_docker_project docker-compose stop
+_docker_project docker-compose rm -f
+exit $test_res

scripts/large_k8s.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+#http://www.apache.org/licenses/LICENSE-2.0.txt
+#
+#
+#Copyright 2016 Intel Corporation
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+set -e
+set -u
+set -o pipefail
+
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+__proj_dir="$(dirname "$__dir")"
+__proj_name="$(basename $__proj_dir)"
+
+. "${__dir}/common.sh"
+
+_info "updating repository submodule with pytest"
+cd ${__proj_dir} && git submodule update --init --recursive
+
+export PROJECT_NAME="${__proj_name}"
+
+_info "execute large test"
+test_result=`python "${__proj_dir}/scripts/test/large.py"`
+exit $test_result