fix a few of the control bash scripts to use GNU coreutils where appl…

…icable (issue #103) A few uses of "grep" and "find" use flags unique to the GNU versions of those tools. As GNU coreutils was already required for a few other utilities I've done the same thing to detect and use ggrep and gfind when needed. As suggested, it might be a good idea to rewrite these to use Python instead to be more portable (although I'll have to take care to make them work with both python 2/3 for various platforms).
cisagov · Jan 21, 2020 · a0cbe89 · a0cbe89
1 parent a26691a
commit a0cbe89
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 26 deletions.
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -7,10 +7,18 @@ if [ -z "$BASH_VERSION" ]; then
   exit 1
 fi
 
+[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath
+[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname
+[[ "$(uname -s)" = 'Darwin' ]] && GREP=ggrep || GREP=grep
+if ! (type "$REALPATH" && type "$DIRNAME" && type "$GREP") > /dev/null; then
+  echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME and $GREP"
+  exit 1
+fi
+
 if docker-compose version >/dev/null 2>&1; then
   DOCKER_COMPOSE_BIN=docker-compose
   DOCKER_BIN=docker
-elif grep -q Microsoft /proc/version && docker-compose.exe version >/dev/null 2>&1; then
+elif $GREP -q Microsoft /proc/version && docker-compose.exe version >/dev/null 2>&1; then
   DOCKER_COMPOSE_BIN=docker-compose.exe
   DOCKER_BIN=docker.exe
 fi
@@ -26,17 +34,11 @@ fi
 
 function filesize_in_image() {
   FILESPEC="$2"
-  IMAGE="$(grep -P "^\s+image:.*$1" docker-compose-standalone.yml | awk '{print $2}')"
+  IMAGE="$($GREP -P "^\s+image:.*$1" docker-compose-standalone.yml | awk '{print $2}')"
   $DOCKER_BIN run --rm --entrypoint /bin/sh "$IMAGE" -c "stat --printf='%s' \"$FILESPEC\""
 }
 
 # force-navigate to Malcolm base directory (parent of scripts/ directory)
-[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath
-[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname
-if ! (type "$REALPATH" && type "$DIRNAME") > /dev/null; then
-  echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME"
-  exit 1
-fi
 SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))"
 pushd "$SCRIPT_PATH/.." >/dev/null 2>&1
 
@@ -59,7 +61,7 @@ read -p "Malcolm Docker images will now be built and/or pulled, force full clean
 CONFIRMATION=${CONFIRMATION:-N}
 
 BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
-MALCOLM_VERSION="$(grep -P "^\s+image:\s*malcolm" "$CONFIG_FILE" | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)"
+MALCOLM_VERSION="$($GREP -P "^\s+image:\s*malcolm" "$CONFIG_FILE" | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)"
 VCS_REVISION="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)"
 
 # MaxMind now requires a (free) license key to download the free versions of their GeoIP databases.
@@ -68,7 +70,7 @@ if [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]; then
   MAXMIND_API_KEY="$MAXMIND_GEOIP_DB_LICENSE_KEY"
 else
   # but default to what they have saved in the docker-compose YML file
-  MAXMIND_API_KEY="$(grep -P "^\s*MAXMIND_GEOIP_DB_LICENSE_KEY\s*:\s" "$CONFIG_FILE" | cut -d: -f2 | tr -d '[:space:]'\'\" | head -n 1)"
+  MAXMIND_API_KEY="$($GREP -P "^\s*MAXMIND_GEOIP_DB_LICENSE_KEY\s*:\s" "$CONFIG_FILE" | cut -d: -f2 | tr -d '[:space:]'\'\" | head -n 1)"
 fi
 
 if [[ $CONFIRMATION =~ ^[Yy] ]]; then

diff --git a/scripts/logs.sh b/scripts/logs.sh
@@ -7,9 +7,17 @@ if [ -z "$BASH_VERSION" ]; then
   exit 1
 fi
 
+[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath
+[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname
+[[ "$(uname -s)" = 'Darwin' ]] && GREP=ggrep || GREP=grep
+if ! (type "$REALPATH" && type "$DIRNAME" && type "$GREP") > /dev/null; then
+  echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME and $GREP"
+  exit 1
+fi
+
 if docker-compose version >/dev/null 2>&1; then
   DOCKER_COMPOSE_BIN=docker-compose
-elif grep -q Microsoft /proc/version && docker-compose.exe version >/dev/null 2>&1; then
+elif $GREP -q Microsoft /proc/version && docker-compose.exe version >/dev/null 2>&1; then
   DOCKER_COMPOSE_BIN=docker-compose.exe
 fi
 
@@ -24,12 +32,6 @@ else
 fi
 
 # force-navigate to Malcolm base directory (parent of scripts/ directory)
-[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath
-[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname
-if ! (type "$REALPATH" && type "$DIRNAME") > /dev/null; then
-  echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME"
-  exit 1
-fi
 SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))"
 pushd "$SCRIPT_PATH/.." >/dev/null 2>&1
 BASE_PATH=`pwd`;
@@ -39,7 +41,7 @@ $DOCKER_COMPOSE_COMMAND ps
 echo ""
 IGNORE_LOG_TEXT="(deprecated|eshealth|update_mapping|throttling index|but no there are no living connections|saved_objects|\b(d|es)?stats\.json|\/_ns_\/nstest\.html|esindices\/list|_cat\/indices|use_field_mapping|reaped unknown pid|Successfully handled GET request for '/'|GET /(_cat/health|api/status|sessions2-).+HTTP/[\d\.].+\b200\b|POST /(d?stats/d?stat|_bulk|fields/field/_search).+HTTP/[\d\.].+\b20[01]\b|POST  HTTP/[\d\.].+\b200\b|POST\s+/server/php/\s+HTTP/\d+\.\d+\"\s+\d+\s+\d+.*:8443/|kibana.+curl.+localhost.+GET /api/status 200)"
 
-$DOCKER_COMPOSE_COMMAND logs -f | grep --line-buffered -Piv "$IGNORE_LOG_TEXT"
+$DOCKER_COMPOSE_COMMAND logs -f | $GREP --line-buffered -Piv "$IGNORE_LOG_TEXT"
 
 echo ""
 

diff --git a/scripts/wipe.sh b/scripts/wipe.sh
@@ -7,6 +7,14 @@ if [ -z "$BASH_VERSION" ]; then
   exit 1
 fi
 
+[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath
+[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname
+[[ "$(uname -s)" = 'Darwin' ]] && FIND=gfind || FIND=find
+if ! (type "$REALPATH" && type "$DIRNAME" && type "$FIND") > /dev/null; then
+  echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME and $FIND"
+  exit 1
+fi
+
 if docker-compose version >/dev/null 2>&1; then
   DOCKER_COMPOSE_BIN=docker-compose
 elif grep -q Microsoft /proc/version && docker-compose.exe version >/dev/null 2>&1; then
@@ -24,12 +32,6 @@ else
 fi
 
 # force-navigate to Malcolm base directory (parent of scripts/ directory)
-[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath
-[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname
-if ! (type "$REALPATH" && type "$DIRNAME") > /dev/null; then
-  echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME"
-  exit 1
-fi
 SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))"
 pushd "$SCRIPT_PATH/.." >/dev/null 2>&1
 
@@ -41,8 +43,8 @@ $SCRIPT_PATH/stop.sh "$CONFIG_FILE" wipe
 
 # completely clean out elasticsearch database and local files
 rm -rf ./elasticsearch/nodes 2>/dev/null
-find ./elasticsearch-backup/ ./zeek-logs/ ./moloch-logs/ ./pcap/ ./moloch-raw/ \( \( -type f -o -type l \) -a ! -name ".gitignore" \) -delete 2>/dev/null
-find ./elasticsearch-backup/logs/ ./zeek-logs/processed/ ./zeek-logs/current/ -mindepth 1 -type d -delete 2>/dev/null
+$FIND ./elasticsearch-backup/ ./zeek-logs/ ./moloch-logs/ ./pcap/ ./moloch-raw/ \( \( -type f -o -type l \) -a ! -name ".gitignore" \) -delete 2>/dev/null
+$FIND ./elasticsearch-backup/logs/ ./zeek-logs/processed/ ./zeek-logs/current/ -mindepth 1 -type d -delete 2>/dev/null
 
 echo "Malcolm has been stopped and its data cleared."
 echo ""