Merge remote-tracking branch 'rgeissert/really-done'

2 files changed, 61 insertions, 9 deletions

diff --git a/bin/ftpsync b/bin/ftpsync
index db8ad55..2c41702 100755
--- a/bin/ftpsync
+++ b/bin/ftpsync
@@ -268,6 +268,10 @@ LOGROTATE=${LOGROTATE:-14}
 LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"}
 # timeout for the lockfile, in case we have bash older than v4 (and no /proc)
 LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
+# sleeping time when an AUIP file is found but is not ours
+UIPSLEEP=${UIPSLEEP:-1200}
+# retries whenever an upstream (or possibly stale) AUIP file is found
+UIPRETRIES=${UIPRETRIES:-3}
 # Do we need another rsync run?
 UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
 # Trace file for mirror stats and checks (make sure we get full hostname)
@@ -475,6 +479,8 @@ fi
 export RSYNC_PASSWORD
 export RSYNC_PROXY
 
+UPDATE_RETRIES=0
+
 while [ -e "${UPDATEREQUIRED}" ]; do
     log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
 
@@ -517,17 +523,54 @@ while [ -e "${UPDATEREQUIRED}" ]; do
 
     # if we want stage2 *or* all
     if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
-        log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+        upstream_uip=false
+        for aupfile in "${TO}/Archive-Update-in-Progress-"*; do
+            case "$aupfile" in
+                "${TO}/Archive-Update-in-Progress-*")
+                    error "Lock file is missing, this should not happen"
+                    ;;
+                "${LOCK}")
+                    :
+                    ;;
+                *)
+                    if [ -f "$aupfile" ]; then
+                        # Remove the file, it will be synced again if
+                        # upstream is still not done
+                        rm -f "$aupfile"
+                    else
+                        log "AUIP file '$aupfile' is not really a file, weird"
+                    fi
+                    upstream_uip=true
+                    ;;
+            esac
+        done
+
+        if [ "xtruex" = "x${upstream_uip}x" ]; then
+            log "Upstream archive update in progress, skipping stage2"
+            if [ ${UPDATE_RETRIES} -lt ${UIPRETRIES} ]; then
+                log "Retrying update in ${UIPSLEEP}"
+                touch "${UPDATEREQUIRED}"
+                UPDATE_RETRIES=$(($UPDATE_RETRIES+1))
+                sleep "${UIPSLEEP}"
+                result=0
+            else
+                error "Update has been retried ${UPDATEREQUIRED} times, aborting"
+                log "Perhaps upstream is still updating or there's a stale AUIP file"
+                result=1
+            fi
+        else
+            log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
 
-        set +e
-        # We are lucky, it worked. Now do step 2 and sync again, this time including
-        # the packages/releases files
-        ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
-            ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
-        result=$?
-        set -e
+            set +e
+            # We are lucky, it worked. Now do step 2 and sync again, this time including
+            # the packages/releases files
+            ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+                ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
+            result=$?
+            set -e
 
-        log "Back from rsync with returncode ${result}"
+            log "Back from rsync with returncode ${result}"
+        fi
     else
         # Fake a good resultcode
         result=0
diff --git a/etc/ftpsync.conf.sample b/etc/ftpsync.conf.sample
index 0b091b6..c8750fe 100644
--- a/etc/ftpsync.conf.sample
+++ b/etc/ftpsync.conf.sample
@@ -89,6 +89,15 @@
 ## synced mirror even if we get multiple pushes in a short timeframe
 #UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
 
+## Number of seconds to sleep before retrying to sync whenever upstream
+## is found to be updating while our update is running
+#UIPSLEEP=1200
+
+## Number of times the update operation will be retried when upstream
+## is found to be updating while our update is running.
+## Note that these are retries, so: 1st attempt + retries = total attempts
+#UIPRETRIES=3
+
 ## The trace file is used by a mirror check tool to see when we last
 ## had a successful mirror sync. Make sure that it always ends up in
 ## project/trace and always shows the full hostname.