[kvmtest]: save sonic vm state for post analysis when test fails (son…

…ic-net#115)
xumia · Jun 18, 2020 · ec5bedd · ec5bedd
1 parent b85681d
commit ec5bedd
Show file tree

Hide file tree

Showing 8 changed files with 31 additions and 12 deletions.
diff --git a/jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile b/jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile
@@ -39,7 +39,7 @@ pipeline {
             post {
                 always {
                     junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**')
+                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**')
                 }
             }
         }

diff --git a/jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile b/jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile
@@ -36,7 +36,7 @@ pipeline {
             post {
                 always {
                     junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**')
+                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**')
                 }
             }
         }

diff --git a/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile b/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile
@@ -66,7 +66,7 @@ pipeline {
 
         always {
             junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-            archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**')
+            archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**')
         }
 
         fixed {

diff --git a/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile b/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile
@@ -85,7 +85,7 @@ sudo cp ../target/sonic-vs.bin /nfs/jenkins/sonic-vs-${JOB_NAME##*/}.${BUILD_NUM
             post {
                 always {
                     junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**')
+                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**')
                 }
             }
         }

diff --git a/jenkins/vs/buildimage-vs-image-test/Jenkinsfile b/jenkins/vs/buildimage-vs-image-test/Jenkinsfile
@@ -65,7 +65,7 @@ pipeline {
     post {
 
         always {
-            archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, sonic-mgmt/tests/results/**')
+            archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, sonic-mgmt/tests/results/**, kvmdump/**')
             junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
         }
 

diff --git a/jenkins/vs/buildimage-vs-image/Jenkinsfile b/jenkins/vs/buildimage-vs-image/Jenkinsfile
@@ -98,7 +98,7 @@ sudo cp ../target/sonic-vs-dbg.bin /nfs/jenkins/sonic-vs-dbg-${JOB_NAME##*/}.${B
             post {
                 always {
                     junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**')
+                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**')
                 }
             }
 

diff --git a/scripts/vs/buildimage-vs-image/runtest.sh b/scripts/vs/buildimage-vs-image/runtest.sh
@@ -1,5 +1,7 @@
 #!/bin/bash -xe
 
+tbname=$1
+
 run_pytest()
 {
     tgname=$1
@@ -24,13 +26,13 @@ mkdir -p .ssh
 cp /data/pkey.txt .ssh/id_rsa
 chmod 600 .ssh/id_rsa
 
-# Refresh virtual switch with vms-kvm-t0 topology
+# Refresh dut in the virtual switch topology
 cd /data/sonic-mgmt/ansible
-./testbed-cli.sh -m veos.vtb -t vtestbed.csv refresh-dut vms-kvm-t0 password.txt
+./testbed-cli.sh -m veos.vtb -t vtestbed.csv refresh-dut $tbname password.txt
 sleep 120
 
 # Create and deploy default vlan configuration (one_vlan_a) to the virtual switch
-./testbed-cli.sh -m veos.vtb -t vtestbed.csv deploy-mg vms-kvm-t0 lab password.txt
+./testbed-cli.sh -m veos.vtb -t vtestbed.csv deploy-mg $tbname lab password.txt
 sleep 180
 
 export ANSIBLE_LIBRARY=/data/sonic-mgmt/ansible/library/
@@ -43,7 +45,7 @@ PYTEST_COMMON_OPTS="--inventory veos.vtb \
                     --user admin \
                     -vvv \
                     --show-capture stdout \
-                    --testbed vms-kvm-t0 \
+                    --testbed $tbname \
                     --testbed_file vtestbed.csv \
                     --disable_loganalyzer \
                     --log-file-level debug"
@@ -87,7 +89,7 @@ popd
 
 # Create and deploy two vlan configuration (two_vlan_a) to the virtual switch
 cd /data/sonic-mgmt/ansible
-./testbed-cli.sh -m veos.vtb -t vtestbed.csv deploy-mg vms-kvm-t0 lab password.txt -e vlan_config=two_vlan_a
+./testbed-cli.sh -m veos.vtb -t vtestbed.csv deploy-mg $tbname lab password.txt -e vlan_config=two_vlan_a
 sleep 180
 
 # Tests to run using two vlan configuration

diff --git a/scripts/vs/buildimage-vs-image/test.sh b/scripts/vs/buildimage-vs-image/test.sh
@@ -4,6 +4,9 @@ echo ${JOB_NAME##*/}.${BUILD_NUMBER}
 
 ls -l
 
+tbname=vms-kvm-t0
+dut=vlab-01
+
 docker login -u $REGISTRY_USERNAME -p $REGISTRY_PASSWD sonicdev-microsoft.azurecr.io:443
 docker pull sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt:latest
 
@@ -23,4 +26,18 @@ cd sonic-mgmt/ansible
 sed -i s:use_own_value:johnar: veos.vtb
 echo abc > password.txt
 cd ../../
-docker run --rm=true -v $(pwd):/data -w /data -i sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt ./scripts/vs/buildimage-vs-image/runtest.sh
+docker run --rm=true -v $(pwd):/data -w /data -i sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt ./scripts/vs/buildimage-vs-image/runtest.sh $tbname
+
+# save dut state if test fails
+if [ $? != 0 ]; then
+    virsh_version=$(virsh --version)
+    if [ $virsh_version == "6.0.0" ]; then
+        rm -rf kvmdump
+        mkdir -p kvmdump
+        virsh save $dut kvmdump/$dut.memdmp
+        virsh dumpxml $dut > kvmdump/$dut.xml
+        img=$(virsh domblklist $dut | grep vda | awk '{print $2}')
+        cp $img kvmdump/$dut.img
+        virsh undefine $dut
+    fi
+fi