Merge pull request fink#37 from dak180/topic/mirror

Changes to the mirror setup docs and path consolidation.

distfiles/finkdistrsync → selfupdate-mirror/fink-dist-rsync

@@ -3,7 +3,7 @@
 # Config
 : "${RSYNCPTH:="rsync://distfiles.master.finkmirrors.net/distfiles/"}"
 FIROOT="/Volumes/src2/fink/distfiles"
-LOCKFILE="/var/run/finkdistrsync.lock"
+LOCKFILE="/var/run/fink-dist-rsync.lock"
 
 usage() {
 	cat > "/dev/stderr" << EOF

selfupdate-mirror/fink-dist-rsync-push

@@ -0,0 +1,102 @@
+#!/bin/sh
+
+# Defaults
+RSYNCPTHS="/Volumes/src2/fink/dist-sync-paths"
+FIROOT="/Volumes/src2/fink/distfiles"
+LOCKFILE="/var/run/fink-dist-rsync-push.lock"
+
+usage() {
+	cat > "/dev/stderr" << EOF
+usage: ${0} [ -l <lockfile> ] [ -o <inputdir> ] [ -i <sync-paths> ]
+Defaults:
+	inputdir: ${FIROOT}
+	lockfile: ${LOCKFILE}
+	sync-paths: ${RSYNCPTHS}
+EOF
+	exit 1
+}
+
+while getopts ":l:o:i:" OPTION; do
+	case "${OPTION}" in
+		l)
+			LOCKFILE="${OPTARG}"
+		;;
+		o)
+			FIROOT="${OPTARG}"
+		;;
+		i)
+			RSYNCPTH="${OPTARG}"
+		;;
+		?)
+			usage
+		;;
+	esac
+done
+
+# Config
+pushDIR="${FIROOT}/push-dist"
+
+# Prevent more than one concurrent update
+lockfile -r "0" -l "28800" "${LOCKFILE}" || exit 0
+
+# Fail if things do not exist
+if [ ! -d "${FIROOT}/distfiles" ]; then
+	rm -f "${LOCKFILE}"
+	exit 1
+fi
+if [ ! -z "${RSYNCPTHS}" ]; then
+	rm -f "${LOCKFILE}"
+	exit 1
+fi
+
+# Make the directories if they do not already exist
+if [ ! -d "${pushDIR}" ]; then
+	mkdir -p "${pushDIR}"
+fi
+
+cd "${FIROOT}"
+
+# Get the list of push sites; # at the start of line is a comment
+RSYNCPTHLIST="$(grep -v '^#' "${RSYNCPTHS}")"
+
+# For each site do a push
+for RSYNCPTH in ${RSYNCPTHLIST}; do
+
+	# Skip blank lines
+	if [ -z "${RSYNCPTH}" ]; then
+		continue
+	fi
+
+	# Note the start time of a new fetch
+	date -u +%s > "${pushDIR}/UPDATE"
+	if ! rsync -azq --timeout=60 --delete-after "${pushDIR}/UPDATE" "${RSYNCPTH}UPDATE"; then
+		echo "error: Fatal inability to contact server ${RSYNCPTH}." >&2
+		continue
+	fi
+
+	# Update from rsync with a 10 min timeout
+	if ! rsync -aq --timeout=600 --delete-after --exclude="TIMESTAMP" --exclude="LOCAL" --exclude="UPDATE" --partial-dir="../distfiles.tmp/" "${FIROOT}/distfiles/" "${RSYNCPTH}"; then
+		echo "error: unable to finish sync with ${RSYNCPTH}." >&2
+		continue
+	fi
+
+	# Push the timestamp file
+	if ! rsync -azq --timeout=60 --delete-after "${FIROOT}/distfiles/TIMESTAMP" "${RSYNCPTH}TIMESTAMP"; then
+		echo "error: unable to finish sync with ${RSYNCPTH}." >&2
+		continue
+	fi
+
+	# Add the local timestamp
+	date -u +%s > "${pushDIR}/LOCAL"
+	if ! rsync -azq --timeout=60 --delete-after "${pushDIR}/LOCAL" "${RSYNCPTH}LOCAL"; then
+		echo "error: Fatal inability to contact server ${RSYNCPTH}." >&2
+		continue
+	fi
+
+done
+
+# Clean up
+rm -f "${LOCKFILE}"
+
+exit 0
+

selfupdate-server/finkcvsup → selfupdate-mirror/fink-info-cvs

@@ -2,7 +2,7 @@
 
 # Config
 FIROOT="/Volumes/src2/fink/selfupdate"
-LOCKFILE="/var/run/finkrsyncup.lock"
+LOCKFILE="/var/run/fink-info-cvs.lock"
 SSHUSER="finkcvs"
 CVSARGS=""
 : "${TIMEOUT:="timeout"}"

selfupdate-server/finkgitup → selfupdate-mirror/fink-info-git

@@ -3,7 +3,7 @@
 # Config
 : "${REPOPTH:="https://github.com/danielj7/fink-dists.git"}"
 FIROOT="/Volumes/src2/fink/selfupdate"
-LOCKFILE="/var/run/finkrsyncup.lock"
+LOCKFILE="/var/run/fink-info-git.lock"
 
 usage() {
 	cat > "/dev/stderr" << EOF
@@ -64,7 +64,7 @@ if ! git fetch origin; then
 	exit 1
 fi
 # Be sure we are on the right branch and then update.
-git checkout master
+git checkout -f master
 if [ "$(git rev-parse master)" = "$(git rev-parse origin/master)" ]; then
 	# If we do not need to update the we are done.
 	date -u +%s > "${FIROOT}/finkinfo/LOCAL"

selfupdate-server/finkrsyncup → selfupdate-mirror/fink-info-rsync

@@ -3,7 +3,7 @@
 # Config
 : "${RSYNCPTH:="rsync://distfiles.master.finkmirrors.net/finkinfo/"}"
 FIROOT="/Volumes/src2/fink/selfupdate"
-LOCKFILE="/var/run/finkrsyncup.lock"
+LOCKFILE="/var/run/fink-info-rsync.lock"
 
 usage() {
 	cat > "/dev/stderr" << EOF

selfupdate-mirror/fink-info-rsync-push

@@ -0,0 +1,103 @@
+#!/bin/sh
+
+# Defaults
+RSYNCPTHS="/Volumes/src2/fink/info-sync-paths"
+FIROOT="/Volumes/src2/fink/selfupdate"
+LOCKFILE="/var/run/fink-info-rsync-push.lock"
+
+usage() {
+	cat > "/dev/stderr" << EOF
+usage: ${0} [ -l <lockfile> ] [ -o <inputdir> ] [ -i <sync-paths> ]
+Defaults:
+	inputdir: ${FIROOT}
+	lockfile: ${LOCKFILE}
+	sync-paths: ${RSYNCPTHS}
+EOF
+	exit 1
+}
+
+while getopts ":l:o:" OPTION; do
+	case "${OPTION}" in
+		l)
+			LOCKFILE="${OPTARG}"
+		;;
+		o)
+			FIROOT="${OPTARG}"
+		;;
+		i)
+			RSYNCPTH="${OPTARG}"
+		;;
+		?)
+			usage
+		;;
+	esac
+done
+
+# Config
+pushDIR="${FIROOT}/push-info"
+
+# Prevent more than one concurrent update
+lockfile -r "0" -l "28800" "${LOCKFILE}" || exit 0
+
+# Fail if things do not exist
+if [ ! -d "${FIROOT}/finkinfo" ]; then
+	rm -f "${LOCKFILE}"
+	exit 1
+fi
+if [ ! -z "${RSYNCPTHS}" ]; then
+	rm -f "${LOCKFILE}"
+	exit 1
+fi
+
+# Make the directories if they do not already exist
+if [ ! -d "${pushDIR}" ]; then
+	mkdir -p "${pushDIR}"
+fi
+
+cd "${FIROOT}"
+
+# Get the list of push sites; # at the start of line is a comment
+RSYNCPTHLIST="$(grep -v '^#' "${RSYNCPTHS}")"
+
+# For each site do a push
+for RSYNCPTH in ${RSYNCPTHLIST}; do
+
+	# Skip blank lines
+	if [ -z "${RSYNCPTH}" ]; then
+		continue
+	fi
+
+	# Note the start time of a new fetch
+	date -u +%s > "${pushDIR}/UPDATE"
+	if ! rsync -azq --timeout=60 --delete-after "${pushDIR}/UPDATE" "${RSYNCPTH}UPDATE"; then
+		echo "error: Fatal inability to contact server ${RSYNCPTH}." >&2
+		continue
+	fi
+
+	# Update from rsync with a 10 min timeout
+	if ! rsync -azq --timeout=600 --delete-after --exclude="TIMESTAMP" --exclude="LOCAL" --exclude="UPDATE" --exclude=".cvsignore" --exclude=".gitignore" --exclude=".git" --exclude=".DS_Store" "${FIROOT}/finkinfo/"  "${RSYNCPTH}"; then
+		rm -f "${LOCKFILE}"
+		exit 1
+	fi
+
+	# Push the timestamp file
+	if ! rsync -azq --timeout=60 --delete-after "${FIROOT}/finkinfo/TIMESTAMP" "${RSYNCPTH}TIMESTAMP"; then
+		echo "error: unable to finish sync with ${RSYNCPTH}." >&2
+		continue
+	fi
+
+	# Add the local timestamp
+	date -u +%s > "${pushDIR}/LOCAL"
+	if ! rsync -azq --timeout=60 --delete-after "${pushDIR}/LOCAL" "${RSYNCPTH}LOCAL"; then
+		echo "error: Fatal inability to contact server ${RSYNCPTH}." >&2
+		continue
+	fi
+
+done
+
+
+# Clean up
+rm -f "${LOCKFILE}"
+
+exit 0
+

selfupdate-server/readme.md → selfupdate-mirror/readme.md

@@ -1,18 +1,21 @@
-# finkinfo mirrors
+# Fink Mirrors
 
 ## Description
 
-These scripts are for setting up finkinfo mirrors.
+These scripts are for setting up Fink mirrors.
 
-These are the mirrors that hold the fink .info and .patch files retrieved by fink when doing a 'fink selfupdate'.
+There are the 'info' mirrors that hold the fink .info and .patch files retrieved by fink when doing a `fink selfupdate` and the 'dist' mirrors that have a copy of all the tarballs referenced by a .info file.
 
 ## Sample rsync setup
 
-This needs to be retrieved via anonymous rsync. These files can be placed anywhere, but make sure your rsync site has the tag `finkinfo` available, and pointing to the directory containing these files.
+This needs to be retrieved via anonymous rsync. These files can be placed anywhere, but make sure your rsync site has the tag `finkinfo` (for info mirrors) and `distfiles` (for dist mirrors) available, and pointing to the directory containing these files.
 
 ```ini
 [finkinfo]
-	path = /Path/src/fink/finkinfo
+	path = /Path/src2/fink/finkinfo
+	comment = Fink .info files
+[distfiles]
+	path = /Path/src2/fink/distfiles
 	comment = Fink .info files
 ```
 
@@ -23,50 +26,66 @@ There are scripts for updating a mirror directly from `cvs` (and eventually `git
 Mirrors must update at an interval between 15 to 90 minutes.
 30 minutes is recommended for mirrors updating via `rsync`.
 Mirrors updating directly from the repository should update as often as they have the resources to do so within the acceptable interval.
+Sites that host both info and dist type mirrors do not have to update both simultaneously or at identical intervals.
 
 The official scripts will timeout on network operations after 10 minutes.
 
-### `finkcvsup`
+### Info Mirror Scripts
+
+#### `fink-info-rsync`
+
+##### Command Line Options
+**`-l`:** Sets the lockfile; `/var/run/fink-info-rsync.lock` by default.
+
+**`-o`:** Sets the output directory; `/Volumes/src2/fink/selfupdate` by default.
+
+##### Environment Variables
+**`RSYNCPTH`:** Sets the uri to sync from; `rsync://distfiles.master.finkmirrors.net/finkinfo/` by default.
+
+#### `fink-info-cvs`
 Requires coreutils to be installed to provide `timeout`.
 
-#### Command Line Options
-**`-l`:** Sets the lockfile; `/var/run/finkrsyncup.lock` by default.
+##### Command Line Options
+**`-l`:** Sets the lockfile; `/var/run/fink-info-cvs.lock` by default.
 
 **`-o`:** Sets the output directory; `/Volumes/src2/fink/selfupdate` by default.
 
 **`-u`:** Sets the ssh user; `finkcvs` by default.
 
 **`-q`:** Makes cvs quiet.
 
-#### Environment Variables
+##### Environment Variables
 **`TIMEOUT`:** Sets the name of the `timeout` command; `timeout` by default.
 
-### `finkrsyncup`
+#### `fink-info-git` (as an example for future use only)
 
-#### Command Line Options
-**`-l`:** Sets the lockfile; `/var/run/finkrsyncup.lock` by default.
+##### Command Line Options
+**`-l`:** Sets the lockfile; `/var/run/fink-info-git.lock` by default.
 
 **`-o`:** Sets the output directory; `/Volumes/src2/fink/selfupdate` by default.
 
-#### Environment Variables
-**`RSYNCPTH`:** Sets the uri to sync from; `rsync://distfiles.master.finkmirrors.net/finkinfo/` by default.
+##### Environment Variables
+**`REPOPTH`:** Sets the uri to sync from; `https://github.com/danielj7/fink-dists.git` by default.
 
-### `finkgitup`
+### Dist Mirror Scripts
 
-#### Command Line Options
-**`-l`:** Sets the lockfile; `/var/run/finkrsyncup.lock` by default.
+#### `fink-dist-rsync`
 
-**`-o`:** Sets the output directory; `/Volumes/src2/fink/selfupdate` by default.
+##### Command Line Options
+**`-l`:** Sets the lockfile; `/var/run/fink-dist-rsync.lock` by default.
+
+**`-o`:** Sets the output directory; `/Volumes/src2/fink/distfiles` by default.
+
+##### Environment Variables
+**`RSYNCPTH`:** Sets the uri to sync from; `rsync://distfiles.master.finkmirrors.net/distfiles/` by default.
 
-#### Environment Variables
-**`REPOPTH`:** Sets the uri to sync from; `https://github.com/danielj7/fink-dists.git` by default.
 
 ## Timestamps
 
 The mirroring network uses three timestamp files to track mirror health.
 
 ### `TIMESTAMP`
-Updated when data is successfully refreshed form the repository.
+Updated when data is successfully refreshed from the repository (or primary sources for dist mirrors); this timestamp is not created by rsync mirrors, only fetched.
 Must always be fetched separately and after the successful retrieval of all other data by rsync driven mirrors.
 
 ### `LOCAL`
@@ -81,6 +100,6 @@ Generally speaking the Fink mirror structure is as follows and please keep in mi
 
 ## Mailing List
 
-If you run (or want to run) a mirror you should subscribe to fink-mirrors-request@lists.sourceforge.net.
+If you run (or want to run) a mirror you should [subscribe](https://lists.sourceforge.net/lists/listinfo/fink-mirrors) to [email protected].
 
 It is important that the person monitoring the list on behalf of a mirror can administrate the mirror should any issues arise.