check missing source files for documents in document check and fix path ordering in history based reindex

[vladak]

As noted on #4317 (comment) , there is another way how the index could be broken - if there are live (i.e. not deleted) documents that are missing the source file. This change augments the document check to report these.

[vladak]

The adjusted document check is needed to test the fix for #4317 , so I added the changes here as well.

[vladak]

I let the the following script run overnight on my laptop (Intel Core i5 - 8 threads, SSD). It ran 36 times, each run had around 100 iterations. This gives me high level of confidence that the duplicates are gone for good.

#!/bin/bash

#
# attempt to reproduce https://github.com/oracle/opengrok/issues/4317
#
# based on git-commit-hopping.sh but with added randomness

# set -x
set -e

repo_url="https://github.com/oracle/solaris-userland/"
initial_rev=32c0d9faed7b049872ca9bd78f9bf3e901cff482	# from 2022

src_root="/var/tmp/src.opengrok-issue-4317"
data_root="/var/tmp/data.opengrok-issue-4317"

# Assumes built OpenGrok.
function run_indexer()
{
	echo "Indexing $1"

        java -cp '/home/vkotal/opengrok-vladak-scratch/distribution/target/dist/*' \
	    -Djava.util.logging.config.file=/var/tmp/logging.properties-opengrok-FINEST_Console \
            org.opengrok.indexer.index.Indexer \
            -c /usr/local/bin/ctags \
	    --economical \
            -H -S -P -s "$src_root" -d "$data_root" \
	    -W "$data_root/config.xml" \
	    >/var/tmp/opengrok-issue-4317-$1.log 2>&1

	java -cp '/home/vkotal/opengrok-vladak-scratch/distribution/target/dist/*' \
            org.opengrok.indexer.index.Indexer \
	    -H \
	    -R "$data_root/config.xml" \
	    --checkIndex documents \
	    >/var/tmp/opengrok-issue-4317-$1.check.log 2>&1
}

function get_next()
{
	ids=( $(git log --oneline --reverse ..origin/master | awk '{ print $1 }') )
	size="${#ids[@]}"
	modulo=16
	if (( size == 0 )); then
		echo ""
		return
	fi
	if (( modulo > size )); then
		modulo=$size
	fi
	n=`expr $RANDOM % $modulo`
	if (( n == 0 )); then
		n=1
	fi
	echo ${ids[$n]}
}

project_root="$src_root/solaris-userland"
if [[ ! -d $src_root ]]; then
	echo "Cloning $repo_url to source root"
	git clone $repo_url "$project_root"
fi

echo "Removing data root"
rm -rf "$data_root"

echo "Removing logs"
rm -f /var/tmp/opengrok-issue*.log

cd "$project_root"
echo "Checking out base rev $initial_rev"
git checkout -q "$initial_rev"
# Establish a common time base line.
find "$src_root/" -type f -exec touch {} \;

run_indexer $initial_rev

while [[ 1 ]]; do
	rev=$(get_next)
	if [[ -z $rev ]]; then
		break
	fi
	echo "Checking out $rev"
	git checkout -q $rev
	# Git does not preserve/restore file time stamps so simulate a git pull.
	# Ideally this should be done only for the "incoming" files, howerver it suffices for this use case.
	find "$src_root/" -type f -exec touch {} \;

	run_indexer $rev
done