Merge branch 'mhop'

* mhop: (28 commits)
  runmirrors
  ftpsync, runmirrors
  runmirrors
  ftpsync
  runmirrors
  common
  runmirrors
  ftpsync tracefile
  fix whitespace indenting to minimize diff
  quote some more arguments
  fix whitespace to minimize diff between master and mhop branches
  readme
  More comments in the mirror sample
  common
  set delay to 600 to be what we tell users in our config file would be default
  runmirrors getopts
  common
  common
  runmirrors
  runmirrors
  ...

2 files changed, 111 insertions, 14 deletions

diff --git a/bin/ftpsync b/bin/ftpsync
index 73b6604..98f80d4 100755
--- a/bin/ftpsync
+++ b/bin/ftpsync
@@ -56,6 +56,7 @@ VERSION="80286"
 # stage1		 Only do stage1 sync
 # stage2		 Only do stage2 sync
 # all			 Do a complete sync
+# mhop           Do a mhop sync, usually additionally to stage1
 # archive:foo    Sync archive foo (if config for foo is available)
 # callback       Call back when done (needs proper ssh setup for this to
 #                work). It will always use the "command" callback:$HOSTNAME
@@ -89,6 +90,9 @@ check_commandline() {
 			sync:all)
 				SYNCALL="true"
 				;;
+			sync:mhop)
+				SYNCMHOP="true"
+				;;
 			*)
 				echo "Unknown option ${1} ignored"
 				;;
@@ -287,6 +291,8 @@ SYNCSTAGE1=${SYNCSTAGE1:-"false"}
 SYNCSTAGE2=${SYNCSTAGE2:-"false"}
 # Do we sync all?
 SYNCALL=${SYNCALL:-"true"}
+# Do we have a mhop sync?
+SYNCMHOP=${SYNCMHOP:-"false"}
 # Do we callback?
 SYNCCALLBACK=${SYNCCALLBACK:-"false"}
 # If we call back we need some more options defined in the config file.
@@ -442,10 +448,14 @@ function mainroutine() {
 	    hook $HOOK
     done
 
-    if [ -d "`dirname "${TO}/${TRACE}"`" ]; then
-        LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
-        echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
-	    echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
+    # We only update our tracefile when we had a stage2 or an all sync.
+    # Otherwise we would update it after stage1 already, which is wrong.
+	if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+        if [ -d "`dirname "${TO}/${TRACE}"`" ]; then
+            LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
+            echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
+	        echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
+        fi
     fi
 
     HOOK=(
@@ -464,10 +474,29 @@ function mainroutine() {
 	rm -f "${LOCK}"
 
     if [ x${HUB} = "xtrue" ]; then
-		# Only trigger slave mirrors if we had a push for stage2 or all
-	    if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+		# Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
+	    if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then
+            RUNMIRRORARGS=""
+            if [ -n "${ARCHIVE}" ]; then
+                # We tell runmirrors about the archive we are running on.
+			    RUNMIRRORARGS="-a ${ARCHIVE}"
+            fi
+			# We also tell runmirrors that we are running it from within ftpsync, so it can change
+			# the way it works with mhop based on that.
+			RUNMIRRORARGS="${RUNMIRRORARGS} -f"
+
+			if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then
+				# This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
+				# not get to this point.
+				# So if that happens, tell runmirrors we are doing mhop
+				RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
+			elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then
+				RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
+			elif [ "xtruex" = "x${SYNCALL}x" ]; then
+				RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
+			fi
 		    log "Trigger slave mirrors"
-		    ${BASEDIR}/bin/runmirrors "${ARCHIVE}"
+		    ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS}
 		    log "Trigger slave done"
 
 		    HOOK=(
diff --git a/bin/runmirrors b/bin/runmirrors
index a79e33d..752e351 100755
--- a/bin/runmirrors
+++ b/bin/runmirrors
@@ -7,7 +7,7 @@ set -u
 # Based losely on existing scripts, written by an unknown number of
 # different people over the years.
 #
-# Copyright (C) 2008 Joerg Jaspert <joerg@debian.org>
+# Copyright (C) 2008, 2009 Joerg Jaspert <joerg@debian.org>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License as
@@ -28,12 +28,68 @@ BASEDIR=${BASEDIR:-"${HOME}"}
 
 NAME="`basename $0`"
 
-# In case we are called with an argument we look for a different configuration.
-CONF=${1:-""}
-if [ -n "${CONF}" ]; then
-	NAME="${NAME}-${CONF}"
+HELP="$0, (C) 2008, 2009 by Joerg Jaspert <joerg@debian.org>\n
+Usage:\n\n
+
+1.) a single parameter with NO leading -.\n
+\t  This will will then be used as the addition for our configfile. Ie. \`$0 security\` will\n
+\t  have us look for ${NAME}-security.{conf,mirror} files.\n\n
+
+2.) using getopt style parameters:\n
+\t -a [NAME]   - Same as 1.) above, used for the config files. Default empty.\n
+\t -k [TYPE]   - Type of push. all, stage2, mhop. Default mhop.\n
+\t -f          - Run from within the mirrorscript ftpsync. Don't use from commandline!\n
+\t -h          - Print this help and exit
+"
+# If we got options, lets see if we use newstyle options, or oldstyle. If oldstyle
+# it will not start with a -. If we find oldstyle we assume its only one, the config
+# name we run on.
+if [ $# -gt 0 ]; then
+    if [ "x${1:0:1}x" != "x-x" ]; then
+        # Yes, does not start with a -, so use it for the config name.
+        CONF=${1:-""}
+        if [ -n "${CONF}" ]; then
+            NAME="${NAME}-${CONF}"
+        fi
+    else
+        # Yeah well, new style, starting with - for getopts
+        while getopts ':a:k:fh' OPTION ; do
+            case $OPTION in
+                a)	CONF="${OPTARG}"
+                    if [ -n "${CONF}" ]; then
+                        NAME="${NAME}-${CONF}"
+                    fi
+                    ;;
+                k)	PUSHKIND="${OPTARG}"
+                    ;;
+                f)	FROMFTPSYNC="true"
+                    ;;
+                h)	echo -e $HELP
+                    exit 0
+                    ;;
+
+                *)  echo "Invalid usage"
+		            echo -e $HELP 
+                    exit 1
+                    ;;
+            esac
+        done
+    fi
 fi
+# Make sure the values are always defined, even if there was no commandline option
+# for them
+# Default config is empty
+CONF=${CONF:-""}
+
+# Set the default to all, if we didnt get told about it. Currently
+# valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1,
+# stage2 - staged push, second phase. Default is mhop.
+PUSHKIND=${PUSHKIND:-"mhop"}
 
+# If we are pushed from within ftpsync. Default false.
+FROMFTPSYNC=${FROMFTPSYNC:-"false"}
+
+########################################################################
 # Read our config file
 . "${BASEDIR}/etc/${NAME}.conf"
 
@@ -80,6 +136,8 @@ HOOK1=${HOOK1:-""}
 HOOK2=${HOOK2:-""}
 HOOK3=${HOOK3:-""}
 
+########################################################################
+
 # Some sane defaults
 cd ${BASEDIR}
 umask 022
@@ -90,7 +148,7 @@ mkdir -p "${LOCKDIR}"
 
 trap 'log "Mirrorpush done" >> ${LOG}; savelog "${LOG}" > /dev/null' EXIT
 
-log "Pushing leaf mirrors" >> ${LOG}
+log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> ${LOG}
 
 HOOK=(
 	HOOKNR=1
@@ -116,13 +174,21 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do
 	if [ "x${MTYPE}x" = "xDELAYx" ]; then
 		# We should wait a bit.
 		if [ -z ${MLNAME} ]; then
-			MLNAME=60
+			MLNAME=600
 		fi
 		log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}"
 		sleep ${MLNAME}
 		continue
 	fi
 
+    # If we are told we have a mhop sync to do and are called from within ftpsync,
+    # we will only look at staged/mhop entries and ignore the rest.
+    if [ "x${PUSHKIND}x" = "xmhopx" ] && [ "x${FROMFTPSYNC}x" = "xtruex" ]; then
+        if [ "x${MTYPE}x" != "xstagedx" ] && [ "x${MTYPE}x" != "xmhopx" ]; then
+            continue
+        fi
+    fi
+
     # Now, MSSHOPT may start with a -. In that case the whole rest of the line is taken
     # as a set of options to give to ssh, we pass it without doing anything with it.
     # If it starts with a 1 or 2 then it will tell us about the ssh protocol version to use,
@@ -172,6 +238,8 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do
 		PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1"
 		PUSHTYPE="${MTYPE}"
 		PUSHARCHIVE=${PUSHARCHIVE}
+		PUSHKIND=${PUSHKIND}
+		FROMFTPSYNC=${FROMFTPSYNC}
 	)
 
 	# And finally, push the mirror