[ci] Fallback travis stack list; fix tests

ci/build.sh

@@ -13,6 +13,9 @@ then
     fi
 fi
 
+# Fetch previously released stacks
+. $script_dir/prefetch.sh
+
 # Allow multiple stacks to be selected
 if [ $# -gt 0 ]
 then
@@ -26,6 +29,5 @@ then
 fi
 
 . $script_dir/lint.sh
-. $script_dir/prefetch.sh
 . $script_dir/package.sh
 . $script_dir/test.sh

ci/env.sh

@@ -10,9 +10,10 @@ export script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 export base_dir=$(cd "${script_dir}/.." && pwd)
 export assets_dir="${script_dir}/assets"
 export build_dir="${script_dir}/build"
+export prefetch_dir="${script_dir}/build/prefetch"
 
 mkdir -p $assets_dir
-mkdir -p $build_dir
+mkdir -p $prefetch_dir
 
 # ENVIRONMENT VARIABLES for controlling behavior of build, package, and release
 
@@ -171,12 +172,16 @@ then
 fi
 
 image_build() {
+    local cmd="docker build"
     if [ "$USE_BUILDAH" == "true" ]; then
-        echo "> ${CI_WAIT_FOR} buildah bud $@"
-        ${CI_WAIT_FOR} buildah bud $@
-    else
-        echo "> ${CI_WAIT_FOR} docker build $@"
-        ${CI_WAIT_FOR} docker build $@
+        cmd="buildah bud"
+    fi
+
+    echo "> ${CI_WAIT_FOR} ${cmd} $@"
+    if ! ${CI_WAIT_FOR} ${cmd} $@
+    then
+      echo "Failed building image"
+      exit 1
     fi
 }
 

ci/ext/pre_list.d/10_travis

@@ -32,7 +32,6 @@ then
             stderr "ERROR: unable to identify changed files in $TRAVIS_COMMIT_RANGE"
             # show original error
             git diff --name-only "$TRAVIS_COMMIT_RANGE"
-            exit 1
         fi
     elif [ $TRAVIS_TAG ]
     then
@@ -68,6 +67,40 @@ then
         fi
     fi
 
+    if [ -z "${STACKS_LIST}" ]
+    then
+        echo "Fall back to stacks that don't match most recent index"
+        printf "  %-50s %-10s %-10s %s\n" "Stack directory" "version" "rebuild"
+        for stack in $(find $base_dir -name stack.yaml)
+        do
+            stack_repo=$(basename $(dirname $(dirname $stack)))
+            stack_name=$(basename $(dirname $stack))
+            stack_version=$(awk '/^version *:/ { gsub("version:","",$NF); gsub("\"","",$NF); print $NF}' $stack)
+            rebuild=true
+
+            if [ $stack_repo = "samples" ]
+            then
+                rebuild=false
+            elif ls ${prefetch_dir} | grep $stack_repo.$stack_name | grep -q $stack_version
+            then
+                # e.g. experimental.java-spring-boot2-liberty.v0.1.7.templates.default.tar.gz
+                rebuild=false
+            elif grep -q $stack_version $build_dir/prefetch-$stack_repo-$stack_name-* 2>/dev/null
+            then
+                # e.g. experimental.java-spring-boot2-liberty.templates.default.tar.gz
+                # use prefetch data
+                rebuild=false
+            fi
+
+            printf "  %-50s %-10s %s\n" "$stack_repo/$stack_name" "$stack_version" "$rebuild"
+            if [ "$rebuild" = "true" ]
+            then
+                export STACKS_LIST+=("$stack_repo/$stack_name")
+            fi
+        done
+    fi
+
+    # Finally, if we still don't have any modified stacks, pick a few
     if [ -z "${STACKS_LIST}" ]
     then
         echo "Choosing a subset of stacks to test with CI/CD changes"

ci/list.sh

@@ -28,6 +28,8 @@ then
     STACKS_LIST=${STACKS_LIST[@]}
 fi
 
+echo "Building stacks: $STACKS_LIST"
+
 # expose environment variable for stacks
 export STACKS_LIST
 

ci/package.sh

@@ -117,8 +117,8 @@ do
                         build=$rebuild_local
                         if [ "$PACKAGE_WHEN_MISSING" = "true" ] &&
                             [ ! -f $assets_dir/$versioned_archive ] &&
-                            [ ! -f $build_dir/prefetch/$old_archive ] &&
-                            [ ! -f $build_dir/prefetch/$versioned_archive ]
+                            [ ! -f $prefetch_dir/$old_archive ] &&
+                            [ ! -f $prefetch_dir/$versioned_archive ]
                         then
                             # force build of template archive if it doesn't exist
                             build=true
@@ -139,7 +139,7 @@ do
                             echo -e "--- Created template archive: $versioned_archive"
 
                             # clean up prefetched resources if they exist
-                            rm -f $build_dir/prefetch/${old_archive%.tar.gz}*
+                            rm -f $prefetch_dir/${old_archive%.tar.gz}*
                         fi
 
                         # Update index yaml based on archive file name (prefer versioned archives)
@@ -156,23 +156,25 @@ do
 
                             echo "      - id: $template_id" >> $index_file
                             echo "        url: $RELEASE_URL/$stack_id-v$stack_version/$versioned_archive" >> $index_file
-                        elif [ -f $build_dir/prefetch/$versioned_archive ]
+                        elif [ -f $prefetch_dir/$versioned_archive ]
                         then
                             # Add references to existing template archive.
                             echo "      - id: $template_id" >> $index_src
                             echo "        url: {{EXTERNAL_URL}}/$versioned_archive" >> $index_src
 
                             echo "      - id: $template_id" >> $index_file
                             echo "        url: $RELEASE_URL/$stack_id-v$stack_version/$versioned_archive" >> $index_file
-                        elif [ -f $build_dir/prefetch/$old_archive ]
+                        elif [ -f $prefetch_dir/$old_archive ]
                         then
+                            verify="$build_dir/prefetch-${repo_name}-${stack_id}-${template_id}"
+
                             # If an archive exists with no version in the name,
                             # check for a prefetch script that can verify it
                             # matches this stack. This helps ensure that the new
                             # stack image
-                            if [ -f $build_dir/prefetch-${stack_id}-${template_id} ]
+                            if [ -f ${verify} ]
                             then
-                                result=$($build_dir/prefetch-${stack_id}-${template_id} ${stack_version})
+                                result=$(${verify} ${stack_version})
                                 if [ "$result" == "ok" ]
                                 then
                                     echo "template ok"
@@ -181,7 +183,7 @@ do
                                     then
                                         stderr "WARNING: checksum for $old_archive doesn't match." \
                                                "         The archive contents don't match what was fetched." \
-                                               "         Pre-fetched checksum is in ${build_dir}/prefetch-${stack_id}-${template_id}"
+                                               "         Pre-fetched checksum is in ${verify}"
                                     else
                                         stderr "WARNING: Version mismatch using $template_id for ${stack_id}-v${stack_version}." \
                                                "         Pre-fetched archive built for ${stack_id}-v${result}."

ci/prefetch.sh

@@ -19,7 +19,6 @@ then
     verify="shasum --status -c -"
 fi
 
-mkdir -p $build_dir/prefetch
 pushd $build_dir/prefetch
 
 if [ -n "${INDEX_LIST}" ]
@@ -44,13 +43,15 @@ then
 
                     # The template id is in the filename: incubator.nodejs-loopback.templates.scaffold.tar.gz
                     template_id=$(echo $filename | sed -E 's|.*templates\.(.*)\.tar\.gz|\1|')
+                    repo=$(echo $filename | cut -d'.' -f1)
 
                     # Create a script that can compare a pre-fetched template version
                     # to the current stack version (read from stack.yaml), e.g.:
                     # ./ci/build/prefetch-stack_id-template_id 0.3.2
                     echo '#!/bin/bash
 
-pushd "'${build_dir}'/prefetch"
+dir=$(pwd)
+cd "'${build_dir}'/prefetch"
 # Fetched from '${url}' on '$(date)'
 # '${x}'
 checksum="'$($create $filename)'"
@@ -65,9 +66,9 @@ then
 else
     echo nomatch
 fi
-popd
-' > ${build_dir}/prefetch-"${stack[0]}-${template_id}"
-                    chmod +x ${build_dir}/prefetch-"${stack[0]}-${template_id}"
+cd $dir
+' > ${build_dir}/prefetch-"${repo}-${stack[0]}-${template_id}"
+                    chmod +x ${build_dir}/prefetch-"${repo}-${stack[0]}-${template_id}"
                 fi
             fi
         done

ci/test.sh

@@ -50,7 +50,7 @@ test_template() {
     if appsody list $index_local | grep -q $stack
     then
         local repo=$index_local
-        export APPSODY_PULL_POLICY=$pull_policy
+        export APPSODY_PULL_POLICY=IFNOTPRESENT
     else
         local repo=$index
         export APPSODY_PULL_POLICY=ALWAYS