Tweaks to building documentation

Dockerfiles/nginx.Dockerfile

@@ -11,6 +11,7 @@
 # first build documentation with jekyll
 FROM ghcr.io/mmguero-dev/jekyll:latest as docbuild
 
+ARG GITHUB_TOKEN
 ARG VCS_REVISION
 ENV VCS_REVISION $VCS_REVISION
 
@@ -20,6 +21,9 @@ WORKDIR /site
 
 # build documentation, remove unnecessary files, then massage a bit to work nicely with NGINX (which will be serving it)
 RUN find /site -type f -name "*.md" -exec sed -i "s/{{[[:space:]]*site.github.build_revision[[:space:]]*}}/$VCS_REVISION/g" "{}" \; && \
+    if [ -n "${GITHUB_TOKEN}" ]; then \
+      export JEKYLL_GITHUB_TOKEN="${GITHUB_TOKEN}"; \
+    fi && \
     docker-entrypoint.sh bundle exec jekyll build && \
     find /site/_site -type f -name "*.md" -delete && \
     find /site/_site -type f -name "*.html" -exec sed -i "s@/\(docs\|assets\)@/readme/\1@g" "{}" \; && \

malcolm-iso/build_via_vagrant.sh

@@ -69,6 +69,8 @@ echo "SSH available." >&2
 cleanup_shared
 mkdir -p "$SCRIPT_PATH"/shared
 [[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot
+echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot
+
 trap cleanup_shared EXIT
 
 if [[ -r "$DOCKER_IMAGES_TGZ" ]]; then

scripts/build.sh

@@ -70,6 +70,7 @@ CONFIRMATION=${CONFIRMATION:-N}
 BUILD_DATE="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
 MALCOLM_VERSION="$($GREP -P "^\s+image:\s*malcolm" "$CONFIG_FILE" | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)"
 VCS_REVISION="$(git rev-parse --short HEAD 2>/dev/null || echo unknown)"
+GITHUB_API_TOKEN="${GITHUB_TOKEN:-}"
 
 [[ ! -f ./auth.env ]] && touch ./auth.env
 
@@ -84,9 +85,9 @@ fi
 
 # build the image(s)
 if [[ $CONFIRMATION =~ ^[Yy] ]]; then
-  $DOCKER_COMPOSE_COMMAND build --progress=plain --force-rm --no-cache --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
+  $DOCKER_COMPOSE_COMMAND build --progress=plain --force-rm --no-cache --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
 else
-  $DOCKER_COMPOSE_COMMAND build --progress=plain --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
+  $DOCKER_COMPOSE_COMMAND build --progress=plain --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
 fi
 
 # we're going to do some validation that some things got pulled/built correctly

scripts/documentation_build.sh

@@ -18,14 +18,24 @@ ENCODING="utf-8"
 
 # parse command-line options
 VERBOSE_FLAG=""
+REVISION="${VCS_REVSION:-$( git rev-parse --short HEAD 2>/dev/null || true )}"
+TOKEN="${GITHUB_TOKEN:-}"
 LOG_BASE_DIR=$(pwd)
-while getopts 'v' OPTION; do
+while getopts 'vr:t:' OPTION; do
   case "$OPTION" in
     v)
       set -x
       VERBOSE_FLAG="-v"
       ;;
 
+    r)
+      REVISION="$OPTARG"
+      ;;
+
+    t)
+      TOKEN="$OPTARG"
+      ;;
+
     ?)
       echo "script usage: $(basename $0) [-v]" >&2
       exit 1
@@ -84,8 +94,17 @@ pushd "$SCRIPT_PATH/.." >/dev/null 2>&1
 cp $VERBOSE_FLAG -r README.md _includes _layouts _config.yml Gemfile docs "$WORKDIR"
 pushd "$WORKDIR" >/dev/null 2>&1
 
+# if the revision commit has been specified, replace references to site.github.build_revision with it
+[[ -n "$REVISION" ]] && find . -type f -name "*.md" -exec sed -i "s/{{[[:space:]]*site.github.build_revision[[:space:]]*}}/$REVISION/g" "{}" \;
+
+# pass GitHub API token through to Jekyll if it's available
+if [[ -n "${TOKEN:-}" ]]; then
+  TOKEN_ARGS=(-e JEKYLL_GITHUB_TOKEN="$TOKEN")
+else
+  TOKEN_ARGS=()
+fi
 # run jekyll docker container to generate HTML version of the documentation
-$DOCKER_BIN run --rm -v "$(pwd)":/site --entrypoint="docker-entrypoint.sh" ghcr.io/mmguero-dev/jekyll:latest bundle exec jekyll build
+$DOCKER_BIN run --rm -v "$(pwd)":/site "${TOKEN_ARGS[@]}" --entrypoint="docker-entrypoint.sh" ghcr.io/mmguero-dev/jekyll:latest bundle exec jekyll build
 
 # clean up some files no longer needed
 find ./_site/ -type f -name "*.md" -delete

sensor-iso/build.sh

@@ -126,6 +126,17 @@ if [ -d "$WORKDIR" ]; then
     cat "$SCRIPT_PATH/shared/environment.chroot" >> ./config/environment.chroot
   echo "PYTHONDONTWRITEBYTECODE=1" >> ./config/environment.chroot
 
+  # format and copy documentation
+  [[ -f "$SCRIPT_PATH/shared/environment.chroot" ]] && \
+    . "$SCRIPT_PATH/shared/environment.chroot"
+  bash "$SCRIPT_PATH/docs/documentation_build.sh" -v -r "${VCS_REVSION:-main}" -t "${GITHUB_TOKEN:-}"
+  mkdir -p ./config/includes.chroot/usr/share/doc
+  cp -r "$SCRIPT_PATH/_site" ./config/includes.chroot/usr/share/doc/hedgehog
+  mkdir -p ./config/includes.chroot/usr/share/fonts/truetype/ubuntu/ ./config/includes.chroot/usr/share/images/hedgehog/ ./config/includes.chroot/usr/share/images/desktop-base/
+  cp "$SCRIPT_PATH/docs/images/hedgehog/logo/"*.png ./config/includes.chroot/usr/share/images/hedgehog/
+  ln -r -s ./config/includes.chroot/usr/share/images/hedgehog/*wallpaper*.png ./config/includes.chroot/usr/share/images/desktop-base/
+  find "$SCRIPT_PATH/docs/images/hedgehog/logo/font/" -type f -name "*.ttf" -exec cp "{}" ./config/includes.chroot/usr/share/fonts/truetype/ubuntu/ \;
+
   # clone and build aide .deb package in its own clean environment (rather than in hooks/)
   bash "$SCRIPT_PATH/shared/aide/build-docker-image.sh"
   docker run --rm -v "$SCRIPT_PATH"/shared/aide:/build aide-build:latest -o /build
@@ -157,14 +168,6 @@ if [ -d "$WORKDIR" ]; then
   cp "$SCRIPT_PATH/arkime"/*.deb ./config/includes.chroot/opt/hedgehog_install_artifacts/
   mv "$SCRIPT_PATH/arkime"/*.deb ./config/packages.chroot/
 
-  # format and copy documentation
-  # TODO
-  pushd "$SCRIPT_PATH/"
-  mkdir -p ./config/includes.chroot/usr/share/fonts/truetype/ubuntu/ ./config/includes.chroot/usr/share/images/hedgehog/ ./config/includes.chroot/usr/share/images/desktop-base/
-  cp "$SCRIPT_PATH/docs/logo/"*.png ./config/includes.chroot/usr/share/images/hedgehog/
-  ln -r -s ./config/includes.chroot/usr/share/images/hedgehog/*wallpaper*.png ./config/includes.chroot/usr/share/images/desktop-base/
-  find "$SCRIPT_PATH/docs/logo/font" -type f -name "*.ttf" -exec cp "{}" ./config/includes.chroot/usr/share/fonts/truetype/ubuntu/ \;
-
   mkdir -p ./config/includes.installer
   cp -v ./config/includes.binary/install/* ./config/includes.installer/
   cp -v ./config/includes.chroot/usr/local/bin/preseed_partman_determine_disk.sh ./config/includes.installer/

sensor-iso/build_via_vagrant.sh

@@ -21,8 +21,14 @@ function vm_execute() {
   vagrant ssh --no-tty --command "$1"
 }
 
-function cleanup_shared {
-  rm -rf "$SCRIPT_PATH"/shared
+function cleanup_shared_and_docs {
+  rm -rf "$SCRIPT_PATH"/shared \
+         "$SCRIPT_PATH"/docs \
+         "$SCRIPT_PATH"/_config.yml \
+         "$SCRIPT_PATH"/_includes \
+         "$SCRIPT_PATH"/_layouts \
+         "$SCRIPT_PATH"/Gemfile \
+         "$SCRIPT_PATH"/README.md
 }
 
 unset FORCE_PROVISION
@@ -70,15 +76,24 @@ until vm_execute 'sudo whoami' | grep -q "root" ; do
 done
 echo "SSH available." >&2
 
-# need to make a temporary local copy of the ../shared directory (and clean it up when we're done)
-cleanup_shared
-cp -r "$SCRIPT_PATH"/../shared "$SCRIPT_PATH"/
+# need to make a temporary local copy of the ../shared directory and files for building documentation (and clean it up when we're done)
+cleanup_shared_and_docs
+cp -r "$SCRIPT_PATH"/../shared \
+      "$SCRIPT_PATH"/../docs \
+      "$SCRIPT_PATH"/../_config.yml \
+      "$SCRIPT_PATH"/../_includes \
+      "$SCRIPT_PATH"/../_layouts \
+      "$SCRIPT_PATH"/../Gemfile \
+      "$SCRIPT_PATH"/../README.md "$SCRIPT_PATH"/
+cp "$SCRIPT_PATH"/../scripts/documentation_build.sh "$SCRIPT_PATH"/docs/
+
 YML_IMAGE_VERSION="$(grep -P "^\s+image:\s*malcolm" "$SCRIPT_PATH"/../docker-compose-standalone.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)"
 [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt
 [[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_LICENSE_KEY" > "$SCRIPT_PATH"/shared/maxmind_license.txt
 [[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot
+echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot
 [[ -d "$SCRIPT_PATH"/../arkime/patch ]] && cp -r "$SCRIPT_PATH"/../arkime/patch "$SCRIPT_PATH"/shared/arkime_patch
-trap cleanup_shared EXIT
+trap cleanup_shared_and_docs EXIT
 
 vm_execute "sudo bash -c \"whoami && cd /sensor-build && pwd && ./build.sh\""
 

sensor-iso/interface/sensor_ctl/control_vars.conf

@@ -16,6 +16,7 @@ export ARKIME_PACKET_ACL=
 export ARKIME_ECS_PROVIDER=arkime
 export ARKIME_ECS_DATASET=session
 
+export DOCUMENTATION_PORT=8420
 export MISCBEAT_PORT=9516
 export FLUENTBIT_METRICS_INTERVAL=30
 export FLUENTBIT_THERMAL_INTERVAL=10

sensor-iso/interface/sensor_ctl/supervisor.d/documentation.conf

@@ -0,0 +1,9 @@
+[program:documentation]
+command=python3 -m http.server --directory /usr/share/doc/hedgehog %(ENV_DOCUMENTATION_PORT)s
+startsecs=5
+startretries=3
+stopasgroup=true
+killasgroup=true
+autostart=true
+directory=/usr/share/doc/hedgehog
+user=sensor