[ci] Add a way to ignore idempotency issues

Vagrantfile

@@ -286,6 +286,7 @@ export CI_JOB_STAGE="#{ENV['CI_JOB_STAGE']}"
 export JANE_TEST_PLAY="#{ENV['JANE_TEST_PLAY']}"
 export JANE_TEST_FACT="#{ENV['JANE_TEST_FACT']}"
 export JANE_TEST_SCRIPT="#{ENV['JANE_TEST_SCRIPT']}"
+export JANE_IGNORE_IDEMPOTENCY="#{ENV['JANE_IGNORE_IDEMPOTENCY']}"
 export JANE_FORCE_TESTS="#{ENV['JANE_FORCE_TESTS']}"
 export JANE_INVENTORY_DIRS="#{ENV['JANE_INVENTORY_DIRS']}"
 export JANE_INVENTORY_GROUPS="#{ENV['JANE_INVENTORY_GROUPS']}"

lib/tests/jane

@@ -66,6 +66,7 @@ readonly JANE_FORCE_TESTS="${JANE_FORCE_TESTS:-}"
 readonly JANE_TEST_PLAY="${JANE_TEST_PLAY:-}"
 readonly JANE_TEST_FACT="${JANE_TEST_FACT:-}"
 readonly JANE_TEST_SCRIPT="${JANE_TEST_SCRIPT:-}"
+readonly JANE_IGNORE_IDEMPOTENCY="${JANE_IGNORE_IDEMPOTENCY:-}"
 readonly JANE_ANSIBLE_CONFIG="${JANE_ANSIBLE_CONFIG:-/vagrant/lib/tests/ansible.cfg}"
 readonly JANE_ANSIBLE_INVENTORY="${JANE_ANSIBLE_INVENTORY:-/vagrant/lib/tests/jane}"
 readonly JANE_INVENTORY_DIRS="${JANE_INVENTORY_DIRS:-common}"
@@ -350,6 +351,7 @@ sub__ansible__playbook () {
     local jane_test_play="${JANE_TEST_PLAY}"
     local jane_test_fact="${JANE_TEST_FACT}"
     local jane_test_script="${JANE_TEST_SCRIPT}"
+    local jane_ignore_idempotency="${JANE_IGNORE_IDEMPOTENCY}"
     local inventory_path
     IFS=':' read -r -a inventory_path <<< "${INVENTORY_PATH}"
     local jane_inventory_dirs
@@ -414,7 +416,11 @@ sub__ansible__playbook () {
     if grep -E -e 'ok=.*changed=.*unreachable=.*failed=.*' /tmp/ansible_idempotent.txt \
         | grep -v -qE -e 'changed=0.*unreachable=0.*failed=0' ; then
         status_msg error "The playbook is NOT IDEMPOTENT"
-        return 1
+        if [ -n "${jane_ignore_idempotency}" ] ; then
+            status_msg warning "Idempotency errors ignored: ${jane_ignore_idempotency}"
+        else
+            return 1
+        fi
     else
         status_msg ok "The playbook is idempotent"
     fi