build_lib.sh

#!/bin/bash

# Library of useful functions for building Crowbar.
# This is sourced by build_crowbar.sh and any other interested parties.

# If we have already been sourced in the current env, don't source us again.
[[ $CROWBAR_BUILD_SOURCED = true ]] && exit 0

[[ $DEBUG ]] && {
    set -x
    export PS4='${BASH_SOURCE}@${LINENO}(${FUNCNAME[0]}): '
}

# Hashes to hold our "interesting" information.
# Key = barclamp name
# Value = whatever interesting thing we are looking for.
declare -A BC_DEPS BC_GROUPS BC_PKGS BC_EXTRA_FILES BC_OS_DEPS BC_GEMS
declare -A BC_REPOS BC_PPAS BC_RAW_PKGS BC_BUILD_PKGS BC_QUERY_STRINGS
declare -A BC_SMOKETEST_DEPS BC_SMOKETEST_TIMEOUTS BC_BUILD_CMDS

# Build OS independent query strings.
BC_QUERY_STRINGS["deps"]="barclamp requires"
BC_QUERY_STRINGS["groups"]="barclamp member"
BC_QUERY_STRINGS["extra_files"]="extra_files"
BC_QUERY_STRINGS["build_cmd"]="build_cmd"
BC_QUERY_STRINGS["os_support"]="barclamp os_support"
BC_QUERY_STRINGS["gems"]="gems pkgs"
BC_QUERY_STRINGS["test_deps"]="smoketest requires"
BC_QUERY_STRINGS["test_timeouts"]="smoketest timeout"

get_barclamp_info() {
    local bc yml_file line query newdeps dep d i
    local new_barclamps=()
    # Pull in interesting information from all our barclamps
    cd "$CROWBAR_DIR"
    for bc in barclamps/*; do
	[[ -d "$bc" ]] || continue
	bc=${bc##*/}
	debug "Reading metadata for $bc barclamp."
	is_barclamp "$bc" || {
	    echo "$bc is not a barclamp, skipping."
	    continue
	}
	yml_file="$CROWBAR_DIR/barclamps/$bc/crowbar.yml"
	[[ $bc = crowbar ]] || BC_DEPS["$bc"]+="crowbar "
	for query in "${!BC_QUERY_STRINGS[@]}"; do
	    while read line; do
		[[ $line = nil ]] && continue
		case $query in
		    deps) is_in "$line "${BC_DEPS["$bc"]} || \
			BC_DEPS["$bc"]+="$line ";;
		    groups) is_in "$line" ${BC_GROUPS["$bc"]} || 
			BC_GROUPS["$line"]+="$bc ";;
		    pkgs|os_pkgs) is_in "$line" ${BC_PKGS["$bc"]} || \
			BC_PKGS["$bc"]+="$line ";;
		    extra_files) BC_EXTRA_FILES["$bc"]+="$line\n";;
		    os_support) BC_OS_SUPPORT["$bc"]+="$line ";;
		    gems) BC_GEMS["$bc"]+="$line ";;
		    repos|os_repos) BC_REPOS["$bc"]+="$line\n";;
		    ppas|os_ppas) [[ $PKG_TYPE = debs ]] || \
			die "Cannot declare a PPA for $PKG_TYPE!"
			BC_REPOS["$bc"]+="ppa $line\n";;
		    build_pkgs|os_build_pkgs) BC_BUILD_PKGS["$bc"]+="$line ";;
		    raw_pkgs|os_raw_pkgs|pkg_sources|os_pkg_sources) BC_RAW_PKGS["$bc"]+="$line ";;
		    test_deps) BC_SMOKETEST_DEPS["$bc"]+="$line ";;
		    os_build_cmd|build_cmd) [[ ${BC_BUILD_CMDS["$bc"]} ]] && \
			die "Only one os_build_cmd stanza per OS per barclamp allowed!"
			BC_BUILD_CMDS["$bc"]="$line";;
		    test_timeouts) BC_SMOKETEST_TIMEOUTS["$bc"]+="$line ";;
		    *) die "Cannot handle query for $query."
		esac
	    done < <("$CROWBAR_DIR/parse_yml.rb" \
		"$yml_file" \
		${BC_QUERY_STRINGS["$query"]} 2>/dev/null)
	done
    done
    cd -

    debug "Analyzing barclamp group membership"
    # If any barclamps need group expansion, do it.
    for bc in "${!BC_DEPS[@]}"; do
	newdeps=''
	for dep in ${BC_DEPS["$bc"]}; do
	    if [[ $dep = @* ]]; then
		[[ ${BC_GROUPS["${dep#@}"]} ]] || \
		    die "$bc depends on group ${dep#@}, but that group does not exist!"
		for d in ${BC_GROUPS["${dep#@}"]}; do
		    is_barclamp "$d" || \
			die "$bc depends on barclamp $d from group ${dep#@}, but $d does not exist!"
		    newdeps+="$d "
		done
	    else
		is_barclamp "$dep" || \
		    die "$bc depends on barclamp $dep, but $dep is not a barclamp!"
		newdeps+="$dep "
	    fi
	done
	BC_DEPS["$bc"]="$newdeps"
    done
    
    # Group-expand barclamps if needed, and unset groups after they are expanded
    for i in "${!BARCLAMPS[@]}"; do
	bc="${BARCLAMPS[$i]}"
	if [[ $bc = @* ]]; then
	    [[ ${BC_GROUPS["${bc#@}"]} ]] || \
		die "No such group ${bc#@}!"
	    BARCLAMPS+=(${BC_GROUPS["${bc#@}"]})
	    unset BARCLAMPS[$i]
	else
	    is_barclamp "$bc" || die "$bc is not a barclamp!"
	fi
    done
    BARCLAMPS=("${BARCLAMPS[@]//@*}")
    
    # Pull in dependencies for the barclamps.
    # Everything depends on the crowbar barclamp, so include it first.
    new_barclamps=("crowbar")
    while [[ t = t ]]; do
	for bc in "${BARCLAMPS[@]}"; do
	    for dep in ${BC_DEPS["$bc"]}; do
		is_in "$dep" "${new_barclamps[@]}" && continue
		new_barclamps+=("$dep")
	    done
	    is_in "$bc" "${new_barclamps[@]}" || new_barclamps+=("$bc")
	done
	[[ ${BARCLAMPS[*]} = ${new_barclamps[*]} ]] && break
	BARCLAMPS=("${new_barclamps[@]}")
    done
}


[[ $CROWBAR_BUILD_PID ]] || export CROWBAR_BUILD_PID=$$
export CLEANUP_LOCK="$CROWBAR_DIR/.cleanup.lock"
cleanup_cmds=()

# Our general cleanup function.  It is called as a trap whenever the 
# build script exits, and it's job is to make sure we leave the local 
# system in the same state we cound it, modulo a few calories of wasted heat 
# and a shiny new .iso.
cleanup() {
    flock -n 70 || exit 1
    # Clean up any stray mounts we may have left behind. 
    # The paranoia with the grepping is to ensure that we do not 
    # inadvertently umount everything.
    if [[ $BASH_SUBSHELL -gt 0 || $BASHPID != $CROWBAR_BUILD_PID ]]; then
	kill -INT $CROWBAR_BUILD_PID
	exit 1
    fi
    for c in "${cleanup_cmds[@]}"; do
	$c || res=1
    done
    
    GREPOPTS=()
    [[ $CACHE_DIR ]] && GREPOPTS=(-e "$CACHE_DIR")
    [[ $IMAGE_DIR && $CACHE_DIR =~ $IMAGE_DIR ]] && GREPOPTS+=(-e "$IMAGE_DIR")
    [[ $BUILD_DIR && $CACHE_DIR =~ $BUILD_DIR ]] && GREPOPTS+=(-e "$BUILD_DIR")
    [[ $CROWBAR_DIR && -d $CROWBAR_DIR/testing ]] && GREPOPTS+=(-e "$CROWBAR_DIR/testing")
    [[ $CHROOT && $CACHE_DIR =~ $CHROOT ]] && GREPOPTS+=(-e "$CHROOT")
    if [[ $GREPOPTS ]]; then
	while read dev fs type opts rest; do
	    sudo umount -d -l "$fs"
	done < <(tac /proc/self/mounts |grep "${GREPOPTS[@]}")
    fi
    # If the build process spawned a copy of webrick, make sure it is dead.
    [[ $webrick_pid && -d /proc/$webrick_pid ]] && kill -9 $webrick_pid
    # clean up after outselves from merging branches, if needed.
    [[ $CI_BARCLAMP ]] &&  {
	if ! in_repo git submodule update -N "barclamps/$CI_BARCLAMP"; then
	    in_ci_barclamp git checkout -f master
	fi
	if ! in_ci_barclamp git branch -D ci-throwaway-branch; then
	    in_ci_barclamp git checkout -f master
	    in_ci_barclamp git branch -D ci-throwaway-branch
	fi
	    
    }
    cd "$CROWBAR_DIR"
    if [[ $THROWAWAY_BRANCH ]]; then
	# Check out the branch we started the build process, and then 
	# nuke whatever throwaway branch we may have created.
	git checkout -f "${CURRENT_BRANCH##*/}" &>/dev/null
	git branch -D "$THROWAWAY_BRANCH" &>/dev/null
    fi
    # If we saved unadded changes, resurrect them.
    [[ $THROWAWAY_STASH ]] && git stash apply "$THROWAWAY_STASH" &>/dev/null
    # Do the same thing as above, but for the build cache instead.
    cd "$CACHE_DIR"
    if ! in_cache git diff-index --cached --quiet HEAD; then
	in_cache git commit -m "Updated by build_crowbar.sh @ $(date) for ${OS_TOKEN}"
	echo "The crowbar build cache has been updated, and the updates have"
	echo "been comitted back to the cache.  Please push any changes."
    fi
    wait
    flock -u 70
    rm "$CROWBAR_DIR/".*.lock
    exit $res
} 70> "$CLEANUP_LOCK"

# Arrange for cleanup to be called at the most common exit points.
trap cleanup 0 INT QUIT TERM

# Test to see if $1 is in the rest of the args.
is_in() {
    local t="(^| )$1( |\$)"
    shift
    [[ $* =~ $t ]]
}

# Run a command in our chroot environment.
in_chroot() { sudo -H /usr/sbin/chroot "$CHROOT" "$@"; }

# A little helper function for doing bind mounts.
bind_mount() {
    [[ -d $2 ]] || mkdir -p "$2"
    grep -q "$2" /proc/self/mounts || sudo mount --bind "$1" "$2"
}

# Write a string with \n translated to a real line break. 
write_lines() { [[ $1 ]] && printf "%b" "$1"; }

# Read base repository information from the *.list files.
read_base_repos() {
    for pkgfile in "$BUILD_DIR/extra/packages/"*.list; do
	[[ -f $pkgfile ]] || continue
	while read line; do
	    line=${line%%#*}
	    [[ $line ]] || continue
	    if [[ $line = repository* ]]; then
		REPOS+=("${line#* }")
	    else
		echo "$line in $pkgfile has been superceded by crowbar.yml metadata." >&2
		die "Please migrate $line into the appropriate barclamp config file."
	    fi
	done <"$pkgfile"
    done
}

# Worker function for all_deps
__all_deps() {
    local dep
    if [[ ${BC_DEPS["$1"]} ]]; then
	for dep in ${BC_DEPS["$1"]}; do
	    is_in "$dep" "${deps[@]}" && continue
	    __all_deps "$dep"
	done
    fi
    is_in "$1" "${deps[@]}" || deps+=("$1")
    return 0
}

# Given a barclamp, echo all of the dependencies of that barclamp
all_deps() {
    local deps=() dep
    __all_deps "$1"
    echo "${deps[*]}"
}	

# A couple of utility functions for comparing version numbers.
num_re='^[0-9]+$'
__cmp() {
    [[ $1 || $2 ]] || return 255 # neither 1 nor 2 or set, we are done
    [[ $1 && ! $2 ]] && return 2 # 1 is set and 2 is not, 1 > 2
    [[ ! $1 && $2 ]] && return 0 # 2 is set and 1 is not, 1 < 2
    local a="$1" b="$2"
    if [[ $a =~ $num_re && $b =~ $num_re ]]; then #both numbers, numeric cmp.
	# make sure leading zeros do not confuse us
	a=${a##0} b=${b##0}
	((${a:=0} > ${b:=0})) && return 2
	(($a < $b)) && return 0
	return 1
    else # string compare
	[[ $a > $b ]] && return 2
	[[ $a < $b ]] && return 0
	return 1
    fi
}

# Compare a version string, handling mixed numeric and alpha sequences
# as appropriate.
vercmp(){
    # $1 = version string of first package
    # $2 = version string of second package
    # Returns 0 if $1 > $2, 1 otherwise.
    # urldecode the version strings.
    local a="$(printf "%b" "${1//%/\\x}")"
    local b="$(printf "%b" "${2//%/\\x}")"
    # prepend a version epoch if one is not already there.
    [[ $a =~ ^[0-9]+: ]] || a="0:$a"
    [[ $b =~ ^[0-9]+: ]] || b="0:$b"
    local ver1=()
    local ver2=()
    local i=0
    # split the version strings into arrays using the usual field splitters
    IFS=':.-_ +' read -rs -a ver1 <<< "$a"
    IFS=':.-_ +' read -rs -a ver2 <<< "$b"
    for ((i=0;;i++)); do
	__cmp "${ver1[$i]}" "${ver2[$i]}"
	case $? in
	    2) return 0;;
	    0) return 1;;
	    255) return 1;;
	esac
    done

}

# Index the pool of packages in the CD.
index_cd_pool() {
    # Scan through our pool to find pkgs we can easily omit.
    local pkgname='' pkg='' cache="$CACHE_DIR/$OS_TOKEN/iso-packages"
    if [[ $ISO_LIBRARY/$ISO -nt $cache ]]; then
	mkdir -p "${cache%/*}"
	> "$cache"
	while read pkg; do
	    [[ -f $pkg ]] && is_pkg "$pkg" || continue
	    pkgname="$(pkg_name "$pkg")"
	    CD_POOL["$pkgname"]="${pkg}"
	    echo "CD_POOL[\"$pkgname\"]=\"${pkg}\"" >> "$cache"
	done < <(find "$(find_cd_pool)" -type f)
    else
	. "$cache"
    fi
}

# Make a chroot environment for package-fetching purposes. 
make_chroot() {
    [[ -f $CHROOT/etc/resolv.conf ]] && return 0
    local bc repo
    debug "Making utility chroot"
    sudo mkdir -p "$CHROOT/$CHROOT_PKGDIR"
    sudo mkdir -p "$CHROOT/$CHROOT_GEMDIR"
    __make_chroot
    
    if [[ $ALLOW_CACHE_UPDATE = true ]]; then
	read_base_repos
        # Add our basic repositories
	add_repos "${REPOS[@]}" || \
	    die "Could not add base repositories for $OS_TOKEN chroot!"

        # Add the repos from the barclamps.
        # We do it here because importing the metadata takes forever
        # if we refresh on every barclamp.
	for bc in "${BARCLAMPS[@]}"; do
	    while read repo; do
		add_repos "$repo"
	    done < <(write_lines "${BC_REPOS[$bc]}")
	done
    fi
    add_offline_repos
    chroot_update
}

stage_pkgs() {
    # $1 = cache to copy from.
    # $2 = location to copy to
    local pkg pkgname pkg_t
    local -A to_copy STAGED_POOL
    while read pkg; do
	# If it is not a package, skip it.
	is_pkg "$pkg" || continue
	pkgname="$(pkg_name "$pkg")"
	# Check to see if it is in the CD pool.
	pkg_t="${CD_POOL["$pkgname"]}"
	# If it is, and the one in the pool is not older than this one,
	# skip it.
	if [[ $pkg_t && -f $pkg_t ]] && ( ! pkg_cmp "$pkg" "$pkg_t" ); then
	    #debug "Skipping copy of ${pkg##*/}, it is on the install media"
	    # if we are shrinking our ISO, make sure this one is in.
	    [[ $SHRINK_ISO = true ]] && INSTALLED_PKGS["$pkgname"]="true"
	    continue
	fi
	# Now check to see if we have already staged it
	pkg_t="${STAGED_POOL["$pkgname"]}"
	if [[ $pkg_t && -f $pkg_t ]]; then
	    # We have already staged it.  Check to see if ours is newer than
	    # the one already staged.
	    if pkg_cmp "$pkg" "$pkg_t"; then
		# We are newer.  Delete the old one, copy us,
		# and update $STAGED_POOL
		#debug "Replacing ${pkg_t##*/} with ${pkg##*/}"
		[[ -f "$pkg_t" ]] && rm -f "$pkg_t"
		[[ ${to_copy["$pkg_t"]} ]] && unset to_copy["$pkg_t"]
		to_copy["$pkg"]="true"
		STAGED_POOL["$pkgname"]="$2/${pkg##*/}"
	    fi
	else
	    # We have not seen this package before.  Copy it.
	    to_copy["$pkg"]="true"
	    cp "$pkg" "$2"
	    STAGED_POOL["$pkgname"]="$2/${pkg##*/}"
	fi
    done < <(find "$1" -type f)
    [[ ${!to_copy[*]} ]] && cp "${!to_copy[@]}" "$2"
}

cache_add() {
    # $1 = file to add.
    # $2 = location to store it in the cache
    cp "$1" "$2" || \
	die "Cannot save $1 in $2!"
    if [[ $CURRENT_CACHE_BRANCH ]]; then
	CACHE_NEEDS_COMMIT=true
	in_cache git add "${2#${CACHE_DIR}/}"
    fi
}

cache_rm() {
    if [[ $CURRENT_CACHE_BRANCH ]]; then
	CACHE_NEEDS_COMMIT=true
	in_cache git rm -f "${1#${CACHE_DIR}/}"
    fi
    rm -f "$1"
}

make_barclamp_pkg_metadata() {
    [[ -d $CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs ]] || return 0
    __barclamp_pkg_metadata_needs_update "$1" || return 0
    [[ $ALLOW_CACHE_METADATA_UPDATE = false ]] && \
	die "Need to update cache metadata for $1, but --no-metadata-update passed."
    debug "Updating package cache metadata for $1"
    make_chroot
    [[ $OS_METADATA_PKGS ]] && {
	chroot_install $OS_METADATA_PKGS
	unset OS_METADATA_PKGS
    }
    sudo mount --bind "$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs" "$CHROOT/mnt"
    __make_barclamp_pkg_metadata "$1"
    sudo umount "$CHROOT/mnt"
}

install_build_packages() {
    chroot_install ${BC_BUILD_PKGS["$1"]}
}

# Update the package cache for a barclamp.
update_barclamp_pkg_cache() {
    # $1 = barclamp we are working with
    local bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs" pkg dest bc
    local -A pkgs
    # Wipe out the packages already in the chroot package directory.
    while read pkg; do
	is_pkg "$pkg" && sudo rm -f "$pkg"
    done < <(find "$CHROOT/$CHROOT_PKGDIR" -type f)
    for bc in $(all_deps "$1"); do
	[[ -d "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." ]] || continue
	sudo cp -a "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." \
	    "$CHROOT/$CHROOT_PKGDIR"
    done
    # Remember what packages we already have.
    while read pkg; do
	is_pkg "$pkg" || continue
	pkgs["$pkg"]="true"
    done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
    [[ ${BC_BUILD_PKGS["$1"]} ]] && install_build_packages "$1"
    chroot_fetch ${BC_PKGS["$1"]} || \
	die "Could not fetch packages required by barclamp $1"
    mkdir -p "$bc_cache"
    while read pkg; do
	is_pkg "$CHROOT/$CHROOT_PKGDIR/$pkg" || continue
	[[ ${pkgs["$pkg"]} = true ]] && continue
	if [[ ${pkg%/*} != '.' ]]; then
	    [[ -d $bc_cache/${pkg%/*} ]] || mkdir -p "$bc_cache/${pkg%/*}"
	    [[ -f $bc_cache/${pkg##*/} ]] && cache_rm "$bc_cache/${pkg##*/}"
	fi 
	cache_add "$CHROOT/$CHROOT_PKGDIR/$pkg" "$bc_cache/$pkg"
    done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
    touch "$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
}

# Update the gem cache for a barclamp
update_barclamp_gem_cache() {
    local -A gems
    local gemname gemver gemopts bc gem
    local bc_cache="$CACHE_DIR/barclamps/$1/gems"

    # Wipe out the caches.
    ( cd "$CHROOT/$CHROOT_GEMDIR" && sudo rm -rf * )
    # if we have deb or gem caches, copy them back in.
    # Make sure we copy the caches for all our dependent barclamps.

   for bc in $(all_deps "$1"); do
	[[ -d "$CACHE_DIR/barclamps/$bc/gems/." ]] && \
	    sudo cp -a "$CACHE_DIR/barclamps/$bc/gems/." \
	    "$CHROOT/$CHROOT_GEMDIR"
    done
    
    while read gem; do
	[[ $gem = *.gem ]] || continue
	gems["$gem"]="true"
    done < <(find "$CHROOT/$CHROOT_GEMDIR" -type f)
        
    # install any build dependencies we need.
    install_build_packages "$1"
    # Grab the gems needed for this barclamp.
    for gem in ${BC_GEMS["$1"]}; do
	if [[ $gem =~ $GEM_RE ]]; then
	    gemname="${BASH_REMATCH[1]}"
	    gemver="${BASH_REMATCH[2]}"
	else
	    gemname="$gem"
	    gemver=''
	fi
	gemopts=(install --no-ri --no-rdoc)
	[[ $gemver ]] && gemopts+=(--version "= ${gemver}")
	[[ $http_proxy ]] && gemopts+=(-p "$http_proxy")
	in_chroot /usr/bin/gem "${gemopts[@]}" "$gemname"
    done

    # Save our updated gems and pkgs in the cache for later.
    mkdir -p "$bc_cache"
     while read gem; do
	[[ $gem = *.gem ]] || continue
	[[ ${gems["$gem"]} = "true" ]] && continue
	cache_add "$gem" "$bc_cache"
    done < <(find "$CHROOT/$CHROOT_GEMDIR" -type f)
}

# Fetch any raw packages we do not already have.
update_barclamp_raw_pkg_cache() {
    local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
    mkdir -p "$bc_cache"
    # Fetch any raw_pkgs we were asked to.
    for pkg in ${BC_RAW_PKGS["$1"]} ${BC_PKG_SOURCES["$1"]}; do
	[[ -f $bc_cache/${pkg##*/} ]] && continue
	echo "Caching $pkg:"
	curl -L -o "$bc_cache/${pkg##*/}" "$pkg"
	[[ $CURRENT_CACHE_BRANCH ]] && in_cache git add "$bc_cache/${pkg##*/}"
    done
    touch "$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
}

# Fetch any bare files that we do not already have.
update_barclamp_file_cache() {
    local dest pkg bc_cache="$CACHE_DIR/barclamps/$1/files"
    # Fetch any extra_pkgs we need.
    mkdir -p "$bc_cache"
    while read pkg; do
	dest=${pkg#* }
	[[ $dest = $pkg ]] && dest=''
	pkg=${pkg%% *}
	[[ -f $bc_cache/files/$dest/${pkg##*/} ]] && continue
	mkdir -p "$bc_cache/$dest"
	echo "Caching $pkg:"
	curl -L -o "$bc_cache/$dest/${pkg##*/}" "$pkg"
	[[ $CURRENT_CACHE_BRANCH ]] && \
	    in_cache git add "$bc_cache/$dest/${pkg##*/}"
    done < <(write_lines "${BC_EXTRA_FILES[$1]}")
}

# Check to see if the barclamp package cache needs update.
barclamp_pkg_cache_needs_update() {
    local pkg pkgname arch bcs=() bc
    local -A pkgs
    
    [[ $need_update = true || ${FORCE_BARCLAMP_UPDATE["$1"]} = true ]] && return 0
    # First, check to see if we have all the packages we need.
    for bc in $(all_deps "$1"); do
	[[ -d "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs" ]] && \
	    bcs+=("$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs")
    done
    if [[ ${bcs[*]} ]]; then
	while read pkg; do
	    is_pkg "$pkg" || continue
	    pkgname="$(pkg_name "$pkg")"
	#debug "$pkgname is cached"
	    pkgs["$pkgname"]="$pkg"
	done < <(find "${bcs[@]}" -type f)
    fi 
    for pkg in ${BC_PKGS["$1"]} ${BC_BUILD_PKGS["$1"]}; do
	[[ $pkg ]] || continue
	for arch in "${PKG_ALLOWED_ARCHES[@]}"; do
	    [[ ${pkgs["$pkg-$arch"]} ]] && continue 2
	    if [[ ${CD_POOL["$pkg-$arch"]} ]]; then
		INSTALLED_PKGS["$pkg-$arch"]="true"
		continue 2
	    fi
	done
	debug "$pkg is not cached, and $1 needs it."
	return 0
    done
    return 1
}

# Check to see if the barclamp gem cache needs an update.
barclamp_gem_cache_needs_update() {
    local pkg pkgname bc 
    local -A pkgs
    # Second, check to see if we have all the gems we need.
    for pkg in ${BC_GEMS["$1"]}; do
	for bc in $(all_deps "$1"); do
	    local bc_cache="$CACHE_DIR/barclamps/$bc/gems"
	    mkdir -p "$bc_cache"
	    [[ $(find "$bc_cache" \
		-name "$pkg*.gem" -type f) = *.gem ]] && continue 2
	done
	return 0    
    done 
    return 1
}

# CHeck to see if we are missing any raw packages.
barclamp_raw_pkg_cache_needs_update() {
    local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
    mkdir -p "$bc_cache"
    # Third, check to see if we have all the raw_pkgs we need.
    for pkg in ${BC_RAW_PKGS["$1"]} ${BC_PKG_SOURCES["$1"]}; do
	[[ -f $bc_cache/${pkg##*/} ]] || return 0
    done
    return 1
}

# Check to see if we are missing any raw files.
barclamp_file_cache_needs_update() {
    local pkg dest bc_cache="$CACHE_DIR/barclamps/$1/files"
    mkdir -p "$bc_cache"
    # Fourth, check to make sure we have all the extra_pkgs we need.
    while read pkg; do
	dest=${pkg#* }
	[[ $dest = $pkg ]] && dest=''
	pkg=${pkg%% *}
	[[ -f $bc_cache/$dest/${pkg##*/} ]] || return 0
    done < <(write_lines "${BC_EXTRA_FILES[$1]}")
    return 1
}

# Some helper functions

# Print a message to stderr and exit.  cleanup will be called.
die() { echo "$(date '+%F %T %z'): $*" >&2; res=1; exit 1; }

# Print a message to stderr and keep going.
debug() { echo "$(date '+%F %T %z'): $*" >&2; }

# Clean up any cruft that we might have left behind from the last run.
clean_dirs() {
    local d=''
    for d in "$@"; do
	(   mkdir -p "$d"
	    cd "$d"
	    sudo rm -rf * )
    done
}

# Verify that the passed name is really a branch in the git repo.
branch_exists() { git show-ref --quiet --verify --heads -- "refs/heads/$1"; }

# Run a git command in the crowbar repo.
in_repo() ( cd "$CROWBAR_DIR"; "$@")

# Get the head revision of a git repository.
get_rev() (
    cd "$1"
    if [[ -d .git ]]; then
	git rev-parse HEAD
    else
	echo "Not a Git Repository"
    fi
)

# Run a git command in the build cache, assuming it is a git repository. 
in_cache() (
    [[ $CURRENT_CACHE_BRANCH ]] || return 
    cd "$CACHE_DIR"
    "$@"
)

# Check to see if something is a barclamp.
is_barclamp() { [[ -f "$CROWBAR_DIR/barclamps/$1/crowbar.yml" ]]; }
in_barclamp() {
    is_barclamp "$1" || die "$1 is not a barclamp"
    (   cd "$CROWBAR_DIR/barclamps/$1"
	shift
	"$@")
}

in_ci_barclamp() {
    [[ $CI_BARCLAMP ]] || die "No continuous integration barclamp!"
    in_barclamp "$CI_BARCLAMP" "$@"
}

# Build our ISO image.
build_iso() (   
    cd "$BUILD_DIR"
    rm -f isolinux/boot.cat
    find -name '.svn' -type d -exec rm -rf '{}' ';' 2>/dev/null >/dev/null
    find . -type f -not -name isolinux.bin -not -name sha1sums \
	-not -path '*/.git/*' | xargs sha1sum -b >sha1sums
    mkdir -p "$ISO_DEST"
	# Save the sha1sums and the build-info files along side the iso.
    cp sha1sums build-info "$ISO_DEST"
    if ! [[ $NO_GENERATE_ISO && $NO_GENERATE_ISO = true ]]; then
	mkisofs -r -V "${VERSION:0:30}" -cache-inodes -J -l -quiet \
	    -b isolinux/isolinux.bin -c isolinux/boot.cat \
	    -no-emul-boot --boot-load-size 4 -boot-info-table \
	    -o "$ISO_DEST/$BUILT_ISO" "$IMAGE_DIR" "$BUILD_DIR"
    fi 
)

# Have the smoketest framework do its thing with the ISO we just made.
test_iso() {
    run_test "$@" || \
	die "$(date '+%F %T %z'): Smoketest of $ISO_DEST/$BUILT_ISO failed."
}

CROWBAR_BUILD_SOURCED=true