add tests + standard test invocation

.dockerignore

@@ -0,0 +1,2 @@
+./Fedora-Cloud-Base-27-1.6.x86_64.qcow2
+./tests

.gitignore

@@ -1,6 +1,9 @@
 Dockerfile
 help/help.md.*
 root/
+tests/artifacts/
+
+*.qcow2
 
 *.py[co]
 *.swp

.travis.yml

@@ -1,12 +1,17 @@
+language: python
+python:
+  - "3.5"
 sudo: required
 services:
   - docker
+before_install:
+  - sudo apt-get -y install acl python3-xattr python3-jinja2 ansible
+  - pip install xattr conu distgen
 script:
   - hooks/pre_build
   # Docker Hub hack
   - sudo cp -av ./Dockerfile.template ./Dockerfile
   - make build
-env:
-  - DG_BINARY="docker run -v $(pwd):/var/dgdir slavek/distgen"
+  - make test
 notifications:
   email: false

Dockerfile.tests

@@ -0,0 +1,2 @@
+FROM docker.io/modularitycontainers/conu
+COPY ./tests /tests

Makefile

@@ -1,9 +1,14 @@
-.PHONY: build run default enumerate-tools upstream fedora-downstream source
+.PHONY: build run default enumerate-tools upstream fedora-downstream source test check
 
 DISTRO := fedora-27-x86_64
 VARIANT := upstream
 DG_BINARY ?= dg
 DG_EXEC = $(DG_BINARY) --max-passes 25 --spec specs/common.yml --multispec specs/multispec.yaml --distro $(DISTRO).yaml --multispec-selector variant=$(VARIANT)
+# set to 1 to enable debugging
+DEBUG_MODE ?= 0
+ifeq ($(DEBUG_MODE), 1)
+	ANSIBLE_EXTRA_ARGS := -vv
+endif
 
 REPOSITORY = $(shell ${DG_EXEC} --template={{spec.repository}})
 
@@ -17,18 +22,21 @@ RENDERED_DOCKERFILE_MD := ./Dockerfile
 
 default: run
 
-$(RENDERED_DOCKERFILE_MD): $(SOURCE_DOCKERFILE_MD)
+root/:
+	mkdir -p ./root
+
+$(RENDERED_DOCKERFILE_MD): $(SOURCE_DOCKERFILE_MD) specs/*
 	$(DG_EXEC) --template $(SOURCE_DOCKERFILE_MD) --output $(RENDERED_DOCKERFILE_MD)
 
-$(RENDERED_README_MD): $(SOURCE_README_MD)
+$(RENDERED_README_MD): $(SOURCE_README_MD) specs/*
 	$(DG_EXEC) --template $(SOURCE_README_MD) --output $(RENDERED_README_MD)
 
-$(RENDERED_HELP_MD): $(SOURCE_HELP_MD) specs/multispec.yaml
+$(RENDERED_HELP_MD): $(SOURCE_HELP_MD) specs/*
 	@# FIXME: current go-md2man can't convert tables :<
 	@# go-md2man -in=${SOURCE_HELP_MD} -out=./root/help.1
 	$(shell TOOLS_CONTAINER_SKIP_ENUMERATION=false $(DG_EXEC) --template $(SOURCE_HELP_MD) --output $(RENDERED_HELP_MD))
 
-source: $(RENDERED_HELP_MD) $(RENDERED_README_MD) $(RENDERED_DOCKERFILE_MD)
+source: root/ $(RENDERED_HELP_MD) $(RENDERED_README_MD) $(RENDERED_DOCKERFILE_MD)
 
 fedora-downstream:
 	make -e source VARIANT="fedora"
@@ -45,6 +53,14 @@ run:
 enumerate-tools:
 	docker run -it -v ${PWD}:/src -e TOOLS_PACKAGES=$(shell $(DG_EXEC) --template="{{spec.packages|join(\",\")}}") --rm $(REPOSITORY) /src/enumerate-tools.py
 
+check: test
+
+test: build
+	make -C tests/ check-local IMAGE_NAME=$(REPOSITORY) ANSIBLE_EXTRA_ARGS=$(ANSIBLE_EXTRA_ARGS)
+
+check-in-vm: build
+	make -C tests/ check-in-vm IMAGE_NAME=$(REPOSITORY) ANSIBLE_EXTRA_ARGS=$(ANSIBLE_EXTRA_ARGS)
+
 clean:
 	rm Dockerfile || :
 	rm root/README.md || :

cccp.yml

@@ -19,7 +19,7 @@ test-skip       : True
 
 # This is path of the script that initiates the user defined tests. It must be able to
 # return an exit code.
-test-script     : null
+test-script     : ./requirements.sh && pytest -vv ./tests/
 
 # This is the path of custom build script.
 build-script    : null

requirements.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+if grep "CentOS Linux 7" /etc/os-release >/dev/null; then
+  cat >/etc/yum.repos.d/virt.repo <<EOF
+[virt7-container-common-candidate]
+name=virt7-container-common-candidate
+baseurl=https://cbs.centos.org/repos/virt7-container-common-candidate/x86_64/os/
+enabled=1
+gpgcheck=0
+EOF
+  # yum remove -y python-chardet  # pip loves overlayfs
+  yum install -y epel-release
+  yum install -y acl nmap-ncat python2-pip python-six pyxattr python2-docker git
+  pip install pytest
+  pip install git+https://github.com/fedora-modularity/conu
+else
+  echo "Unsupported distro"
+  exit 1
+fi

tests/Makefile

@@ -0,0 +1,16 @@
+.PHONY: check-local check-in-vm
+
+IMAGE_NAME := ""
+VM_IMAGE_NAME := Fedora-Cloud-Base-27-1.6.x86_64.qcow2
+VM_IMAGE_PATH = ../$(VM_IMAGE_NAME)
+INVENTORY := /usr/share/ansible/inventory/standard-inventory-qcow2
+ANSIBLE_EXTRA_ARGS ?=
+
+check-local:
+	ansible-playbook $(ANSIBLE_EXTRA_ARGS) -e subject=$(IMAGE_NAME) ./local.yml
+
+check-in-vm: $(VM_IMAGE_PATH)
+	TEST_SUBJECTS=$(VM_IMAGE_PATH) ansible-playbook $(ANSIBLE_EXTRA_ARGS) -e ansible_python_interpreter=/usr/bin/python3 -e subject=$(IMAGE_NAME) -i $(INVENTORY) -e setup=true -e vm_image=$(VM_IMAGE) ./in-vm.yml
+
+$(VM_IMAGE_PATH):
+	curl -L -o $(VM_IMAGE_PATH) https://download.fedoraproject.org/pub/fedora/linux/releases/27/CloudImages/x86_64/images/$(VM_IMAGE_NAME)

tests/ansible.cfg

@@ -0,0 +1,4 @@
+[defaults]
+inventory = ./inventory
+retry_files_enabled = false
+# roles_path = </path/to/the/repo>/roles

tests/in-vm.yml

@@ -0,0 +1,82 @@
+# support use cases:
+#  * testing an image built locally in current environment (set pull variable to false)
+#  * testing an image built locally inside a VM (supply vm_image variable)
+#  * testing an image present in registry in current environment (set pull to true)
+#  * testing an image present in registry inside a VM (set pull and vm_image variables)
+---
+- name: Integration tests for tools container image executed in current environment
+  hosts: localhost
+  vars:
+    # don't pull the test subject by default
+    pull: false
+
+    # don't set up the environment by default (instal and start container runtime)
+    setup: false
+
+    required_packages:
+     - python3-conu
+     - python3-pytest
+
+    # tests to be invoked (this is utilized by basic standard test role)
+    tests:
+    # test suites = directories, where the tests live
+    - integration
+
+    # our test subject
+    subject: ""
+
+    # path where the test artifacts will be stored - logs
+    artifacts: "{{ playbook_dir }}/artifacts/"
+
+  tasks:
+  - name: prepare the environment to run tests
+    block:
+    - name: Install the container engine
+      package:
+        name: docker
+        state: present
+      become: true
+    - name: Start the container engine
+      systemd:
+        name: docker
+        state: started
+      become: true
+    when: setup
+
+  - name: Pull the test subject (=container image)
+    command: docker pull {{ subject }}
+    when: pull
+
+  - name: Copy test subject from host inside the VM
+    block:
+    # FIXME: make this configurable
+    - name: Create temporary directory for the image
+      tempfile:
+        state: directory
+      register: tmp
+    - name: Save the image to a file
+      command: 'docker save -o {{ tmp.path + "/image.tar.gz" }} {{ subject }}'
+    - name: Copy the image from host to the target
+      # synchronize is so unreliable
+      synchronize:
+        src: '{{ tmp.path + "/image.tar.gz" }}'
+        dest: '/tmp/'
+        mode: push
+        ssh_args: "-o UserKnownHostsFile=/dev/null -i {{ ansible_ssh_private_key_file }}"
+    - file:
+        state: absent
+        path: "{{ tmp.path }}"
+    when: not pull
+    delegate_to: localhost
+
+  - block:
+    - name: Load the image on the target into dockerd
+      command: 'docker load -i /tmp/image.tar.gz'
+    - file:
+        state: absent
+        path: "/tmp/image.tar.gz"
+    when: not pull
+
+  - name: Execute the role which performs testing
+    import_role:
+      name: standard-test-basic

tests/integration/runtest.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+pytest-3 -vv ./test_container.py

tests/integration/test_container.py

@@ -0,0 +1,79 @@
+#!/usr/bin/python3
+
+import logging
+import os
+
+import conu
+
+import pytest
+
+
+IMAGE = os.environ.get("IMAGE_NAME", "docker.io/modularitycontainers/tools")
+TAG = os.environ.get("IMAGE_TAG", "latest")
+
+
+@pytest.fixture(scope="module")
+def container(request):
+    with conu.DockerBackend(logging_level=logging.DEBUG) as backend:
+        im = backend.ImageClass(IMAGE, tag=TAG)
+        b = conu.DockerRunBuilder(command=["sleep", "infinity"])
+        b.options += [
+            "--net", "host",
+            "--pid=host",
+            "--ipc", "host",
+            "-it",
+            "--privileged",
+            "-v", "/run:/run",
+            "-v", "/:/host",
+            "-v", "/var/log:/var/log",
+        ]
+        machine_id_path = "/etc/machine-id"
+        if os.path.exists(machine_id_path):
+            b.options += [
+                "-v", "%s:%s" % (machine_id_path, machine_id_path)
+            ]
+        localtime_path = "/etc/localtime"
+        if os.path.exists(localtime_path):
+            b.options += [
+                "-v", "%s:%s" % (localtime_path, localtime_path)
+            ]
+        container = im.run_via_binary(b)
+    yield container
+    container.stop()
+    container.delete()
+
+
+class TestContainer:
+    def test_ethtool(self, container):
+        # with self.container.mount() as fs:
+        #     networks_devices = os.listdir(fs.p("/sys/class/net"))
+        networks_devices = ["lo"]
+        for device in networks_devices:
+            container.execute(["ethtool", device])
+        with pytest.raises(conu.ConuException):
+            container.execute(["ethtool", "quantum-teleport"])
+
+    def test_netstat(self, container):
+        container.execute(["netstat"])
+
+    def test_ss(self, container):
+        container.execute(["ss"])
+
+    def test_pstack(self, container):
+        container.execute(["pstack", "1"])
+
+    def test_nstat(self, container):
+        container.execute(["nstat"])
+
+    def test_numastat(self, container):
+        container.execute(["numastat"])
+
+    def test_pmap(self, container):
+        container.execute(["pmap", "1"])
+
+    def test_strace(self, container):
+        container.execute(["strace", "-V"])
+
+
+if __name__ == '__main__':
+    pytest.main()

tests/inventory

@@ -0,0 +1 @@
+localhost ansible_connection=local

tests/local.yml

@@ -0,0 +1,61 @@
+# support use cases:
+#  * testing an image built locally in current environment (set pull variable to false)
+#  * testing an image present in registry in current environment (set pull to true)
+---
+- name: Integration tests for tools container image executed in current environment
+  hosts: localhost
+  vars:
+    # don't pull the test subject by default
+    pull: false
+
+    # don't set up the environment by default (install and start container runtime)
+    setup: false
+
+    required_packages:
+     - python3-conu
+     - python3-pytest
+
+    tests:
+    - integration
+
+    # our test subject
+    subject: ""
+
+    # path where the test artifacts will be stored - logs
+    artifacts: "{{ playbook_dir }}/artifacts/"
+
+  tasks:
+  - name: prepare the environment to run tests
+    block:
+    - name: Install the container engine
+      package:
+        name: docker
+        state: present
+      become: true
+    - name: Start the container engine
+      systemd:
+        name: docker
+        state: started
+      become: true
+    when: setup
+
+  - name: Pull the test subject (=container image)
+    command: docker pull {{ subject }}
+    when: pull
+
+  - block:
+    # should this be configurable?
+    - name: Create temp dir to store tests
+      tempfile:
+        state: directory
+      register: tmp_tests
+    - name: Execute the role which performs testing
+      import_role:
+        name: standard-test-basic
+      vars:
+        tenv_workdir: "{{ tmp_tests.path }}"
+    always:
+    - name: delete the temp dir
+      file:
+        path: "{{ tmp_tests.path }}"
+        state: absent