archive_url

#! /bin/sh

# archive_url - save not yet archived URLs in the Wayback Machine
# Copyright (C) 2021-2025 Erik Auerswald
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

set -u
set -e

PROG=archive_url
VERSION='2025-01-11-01'
FORCE=0
DRYRUN=0
QUIET=0
VERBOSE=0
WAIT=''
WAITTIME=5
JITTERWAIT=0
WGETVERB='-nv'
CHECK='https://archive.org/wayback/available?url='
ARCHIVE='https://web.archive.org/save/'
EXIT_CODE=0

print_copyright()
{
  echo 'Copyright (C) 2021-2025 Erik Auerswald'
}

print_version()
{
  printf -- '%s version %s\n' "${PROG}" "${VERSION}"
}

print_usage()
{
  printf -- '%s { -h | -V | -L }\n' "$PROG"
  printf -- '%s [-f] [-n] [-q] [-v] [-w TIME] [-r] [URL...]\n' "$PROG"
}

print_license()
{
  cat <<EOL
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
EOL
}

print_help()
{
  print_version
  print_copyright
  echo
  print_license
  cat <<EOD

Check if the given URLs are already archived in the Wayback Machine
(web.archive.org), and attempt to archive those that are not.

URLs can be provided as arguments.
Without arguments, URLs are read from standard input.

EOD
  print_usage
  cat <<EOH
Options:
  -h       print this help and exit
  -V       print version information and exit
  -L       print license and exit
  -f       save URLs without checking
  -n       check URLs only, but do not archive (dry run)
  -q       suppress error messages and wget output
  -v       print URL that is processed next
  -w TIME  wait at least TIME seconds after each URL (default $WAITTIME)
  -r       randomize wait time between URLs
EOH
}

verb_no_nl() {
  test "$VERBOSE" -eq 1 || return 0
  printf -- '%s' "$*"
}

verb_nl() {
  test "$VERBOSE" -eq 1 || return 0
  printf -- '%s\n' "$*"
}

err() {
  EXIT_CODE=1
  test "$QUIET" -eq 0 || return 0
  printf -- '%s: %s ERROR: %s\n' \
            "$PROG" "$(date '+%Y-%m-%d %H:%M:%S')" "$*" 1>&2
}

add_jitter() {
  if command -v 'shuf' >/dev/null; then
    WAITINT=${WAITTIME%%.*}
    WAITMAX=$((WAITINT / 5))
    JITTERINT=$(shuf -i"0-${WAITMAX}" -n1)
    JITTERFRAC=$(shuf -i"0-999" -n1)
    echo "$((WAITINT + JITTERINT)).${JITTERFRAC}"
  else
    echo "$WAITTIME"
  fi
}

do_wait() {
  REALWAIT="$WAITTIME"
  test "$JITTERWAIT" -eq 1 && REALWAIT=$(add_jitter)
  verb_no_nl "$(date '+%Y-%m-%d %H:%M:%S')" 'Waiting' "$REALWAIT" 'seconds ...'
  sleep -- "$REALWAIT"
  verb_nl ' done'
}

while getopts ':hVLfnqvw:r' OPT; do
  case "$OPT" in
    'h') print_help; exit 0;;
    'V') print_version; print_copyright; exit 0;;
    'L') print_version; print_copyright; echo; print_license; exit 0;;
    'f') FORCE=1;;
    'n') DRYRUN=1;;
    'q') QUIET=1 WGETVERB='-q';;
    'v') VERBOSE=1;;
    'w') WAITTIME="$OPTARG";;
    'r') JITTERWAIT=1;;
    '?') err "unknown option '-$OPTARG'"; print_usage; exit 1;;
    ':') err "argument required for option '-$OPTARG'"; print_usage; exit 1;;
    *)   err 'getopts() failure'; exit 1;;
  esac
done
shift $((OPTIND - 1))

if ! printf -- '%s\n' "$WAITTIME" | grep -Eq '^([0-9]+(\.[0-9]*)?|\.[0-9]+)$'
then
  err "TIME must be a decimal number"
  exit 1
fi

check_and_archive_url()
{
  URL="$1"
  if test "$FORCE" -eq 0; then
    verb_no_nl "$(date '+%Y-%m-%d %H:%M:%S') Checking ${URL} ..."
    STATUS="$(wget -q -O- -- "${CHECK}${URL}" | jq '.archived_snapshots != {}')"
  else
    verb_no_nl "$(date '+%Y-%m-%d %H:%M:%S') Forcing ${URL} ..."
    STATUS='false'
  fi
  if test "$STATUS" = 'true'; then
    verb_nl ' archived'
  else
    if test "$DRYRUN" -eq 0; then
      verb_nl ' triggering archiving ...'
      wget "$WGETVERB" '-t1' '-O/dev/null' -- "${ARCHIVE}${URL}" ||
        err "error archiving URL (Wget exit code $?)"
    else
      verb_nl ' not archived (dry run)'
    fi
  fi
}

if test "$#" -eq 0; then
  while read -r URL; do
    test -n "$WAIT" && do_wait
    check_and_archive_url "$URL"
    WAIT='true'
  done
else
  for URL in "$@"; do
    test -n "$WAIT" && do_wait
    check_and_archive_url "$URL"
    WAIT='true'
  done
fi

exit "$EXIT_CODE"