From 9d7921a36eabd23d2fd9e842fae01bfe01e8a69c Mon Sep 17 00:00:00 2001 From: Raphael Geissert Date: Thu, 26 Apr 2012 22:28:56 -0500 Subject: Only update the trace file if no AUIP file is found It is otherwise possible for a mirror to be synced while the upstream is not yet done syncing and for the site trace to be updated regardless. --- bin/ftpsync | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/bin/ftpsync b/bin/ftpsync index 1ae7b81..9cfe35e 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -523,7 +523,21 @@ done # We only update our tracefile when we had a stage2 or an all sync. # Otherwise we would update it after stage1 already, which is wrong. if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - if [ -d "$(dirname "${TO}/${TRACE}")" ]; then + should_update=true + for aupfile in "${TO}/Archive-Update-in-Progress-"*; do + case "$aupfile" in + "${TO}/Archive-Update-in-Progress-*") + : + ;; + "${LOCK}") + : + ;; + *) + should_update=false + ;; + esac + done + if $should_update && [ -d "$(dirname "${TO}/${TRACE}")" ]; then LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" -- cgit v1.2.3 From 78ac9758104d9f23e3d8c346bae865902e7f36dd Mon Sep 17 00:00:00 2001 From: Raphael Geissert Date: Thu, 26 Apr 2012 23:38:17 -0500 Subject: Only perform the second stage if upstream is not updating Instead of not updating the trace file, skip the second stage so that the mirror is kept in a consistent state. --- bin/ftpsync | 54 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index 9cfe35e..646e9d8 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -487,17 +487,37 @@ while [ -e "${UPDATEREQUIRED}" ]; do # if we want stage2 *or* all if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + upstream_uip=false + for aupfile in "${TO}/Archive-Update-in-Progress-"*; do + case "$aupfile" in + "${TO}/Archive-Update-in-Progress-*") + : + ;; + "${LOCK}") + : + ;; + *) + upstream_uip=true + ;; + esac + done - set +e - # We are lucky, it worked. Now do step 2 and sync again, this time including - # the packages/releases files - ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error" - result=$? - set -e + if $upstream_uip; then + log "Upstream archive update in progress, skipping stage2 (ignore rsync error below)" + result=1 + else + log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + + set +e + # We are lucky, it worked. Now do step 2 and sync again, this time including + # the packages/releases files + ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error" + result=$? + set -e - log "Back from rsync with returncode ${result}" + log "Back from rsync with returncode ${result}" + fi else # Fake a good resultcode result=0 @@ -523,21 +543,7 @@ done # We only update our tracefile when we had a stage2 or an all sync. # Otherwise we would update it after stage1 already, which is wrong. if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - should_update=true - for aupfile in "${TO}/Archive-Update-in-Progress-"*; do - case "$aupfile" in - "${TO}/Archive-Update-in-Progress-*") - : - ;; - "${LOCK}") - : - ;; - *) - should_update=false - ;; - esac - done - if $should_update && [ -d "$(dirname "${TO}/${TRACE}")" ]; then + if [ -d "$(dirname "${TO}/${TRACE}")" ]; then LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" -- cgit v1.2.3 From f56f99e06c75697526dfbcaf2594461d6f489922 Mon Sep 17 00:00:00 2001 From: Raphael Geissert Date: Thu, 26 Apr 2012 23:40:55 -0500 Subject: Better error reporting --- bin/ftpsync | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ftpsync b/bin/ftpsync index 646e9d8..85bb9c7 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -491,7 +491,7 @@ while [ -e "${UPDATEREQUIRED}" ]; do for aupfile in "${TO}/Archive-Update-in-Progress-"*; do case "$aupfile" in "${TO}/Archive-Update-in-Progress-*") - : + error "Lock file is missing, this should not happen" ;; "${LOCK}") : -- cgit v1.2.3 From 521dc27e941f7af8ee35c264e8dc5b352d6beb33 Mon Sep 17 00:00:00 2001 From: Raphael Geissert Date: Thu, 26 Apr 2012 23:56:04 -0500 Subject: Retry the sync after a delay if an upstream AUIP file is found --- bin/ftpsync | 9 +++++++-- etc/ftpsync.conf.sample | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index 85bb9c7..559200b 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -268,6 +268,8 @@ LOGROTATE=${LOGROTATE:-14} LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"} # timeout for the lockfile, in case we have bash older than v4 (and no /proc) LOCKTIMEOUT=${LOCKTIMEOUT:-3600} +# sleeping time when an AUIP file is found but is not ours +UIPSLEEP=${UIPSLEEP:-1200} # Do we need another rsync run? UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" # Trace file for mirror stats and checks (make sure we get full hostname) @@ -503,8 +505,11 @@ while [ -e "${UPDATEREQUIRED}" ]; do done if $upstream_uip; then - log "Upstream archive update in progress, skipping stage2 (ignore rsync error below)" - result=1 + log "Upstream archive update in progress, skipping stage2" + log "Retrying update in ${UIPSLEEP}" + touch "${UPDATEREQUIRED}" + sleep "${UIPSLEEP}" + result=0 else log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" diff --git a/etc/ftpsync.conf.sample b/etc/ftpsync.conf.sample index e3fcc63..9e8cc65 100644 --- a/etc/ftpsync.conf.sample +++ b/etc/ftpsync.conf.sample @@ -89,6 +89,10 @@ ## synced mirror even if we get multiple pushes in a short timeframe #UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" +## Number of seconds to sleep before retrying to sync whenever upstream +## is found to be updating while our update is running +#UIPSLEEP=1200 + ## The trace file is used by a mirror check tool to see when we last ## had a successful mirror sync. Make sure that it always ends up in ## project/trace and always shows the full hostname. -- cgit v1.2.3 From dbb0bdd950dccccdd0f9a3d43c656c2a03e4a56a Mon Sep 17 00:00:00 2001 From: Raphael Geissert Date: Mon, 30 Apr 2012 16:55:44 -0500 Subject: Change the code style to match that of the rest of the code --- bin/ftpsync | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index 559200b..c73a0ee 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -494,17 +494,17 @@ while [ -e "${UPDATEREQUIRED}" ]; do case "$aupfile" in "${TO}/Archive-Update-in-Progress-*") error "Lock file is missing, this should not happen" - ;; + ;; "${LOCK}") : - ;; + ;; *) upstream_uip=true - ;; + ;; esac done - if $upstream_uip; then + if [ "xtruex" = "x${upstream_uip}x" ]; then log "Upstream archive update in progress, skipping stage2" log "Retrying update in ${UIPSLEEP}" touch "${UPDATEREQUIRED}" -- cgit v1.2.3 From 361ba6bce1a3fabae9da0ab2b97f2cc7360da1d3 Mon Sep 17 00:00:00 2001 From: Raphael Geissert Date: Tue, 1 May 2012 00:14:06 -0500 Subject: Limit the number of update retries In case something went wrong somewhere and upstream's update stalled or there are stale AUIP files, this will prevent ftpsync from retrying without an eventual error message. --- bin/ftpsync | 19 +++++++++++++++---- etc/ftpsync.conf.sample | 5 +++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index c73a0ee..b72b435 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -270,6 +270,8 @@ LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"} LOCKTIMEOUT=${LOCKTIMEOUT:-3600} # sleeping time when an AUIP file is found but is not ours UIPSLEEP=${UIPSLEEP:-1200} +# retries whenever an upstream (or possibly stale) AUIP file is found +UIPRETRIES=${UIPRETRIES:-3} # Do we need another rsync run? UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" # Trace file for mirror stats and checks (make sure we get full hostname) @@ -447,6 +449,8 @@ fi export RSYNC_PASSWORD export RSYNC_PROXY +UPDATE_RETRIES=0 + while [ -e "${UPDATEREQUIRED}" ]; do log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" @@ -506,10 +510,17 @@ while [ -e "${UPDATEREQUIRED}" ]; do if [ "xtruex" = "x${upstream_uip}x" ]; then log "Upstream archive update in progress, skipping stage2" - log "Retrying update in ${UIPSLEEP}" - touch "${UPDATEREQUIRED}" - sleep "${UIPSLEEP}" - result=0 + if [ ${UPDATE_RETRIES} -lt ${UIPRETRIES} ]; then + log "Retrying update in ${UIPSLEEP}" + touch "${UPDATEREQUIRED}" + UPDATE_RETRIES=$(($UPDATE_RETRIES+1)) + sleep "${UIPSLEEP}" + result=0 + else + error "Update has been retried ${UPDATEREQUIRED} times, aborting" + log "Perhaps upstream is still updating or there's a stale AUIP file" + result=1 + fi else log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" diff --git a/etc/ftpsync.conf.sample b/etc/ftpsync.conf.sample index 9e8cc65..e3f5cf5 100644 --- a/etc/ftpsync.conf.sample +++ b/etc/ftpsync.conf.sample @@ -93,6 +93,11 @@ ## is found to be updating while our update is running #UIPSLEEP=1200 +## Number of times the update operation will be retried when upstream +## is found to be updating while our update is running. +## Note that these are retries, so: 1st attempt + retries = total attempts +#UIPRETRIES=3 + ## The trace file is used by a mirror check tool to see when we last ## had a successful mirror sync. Make sure that it always ends up in ## project/trace and always shows the full hostname. -- cgit v1.2.3 From d213b86389ee6425cd4f51ea63e68eea1c9bb988 Mon Sep 17 00:00:00 2001 From: Raphael Geissert Date: Tue, 10 Jul 2012 00:46:31 -0500 Subject: Remove upstream's AUIP file to avoid a deadlock --- bin/ftpsync | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/ftpsync b/bin/ftpsync index b72b435..951aed7 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -503,6 +503,13 @@ while [ -e "${UPDATEREQUIRED}" ]; do : ;; *) + if [ -f "$aupfile" ]; then + # Remove the file, it will be synced again if + # upstream is still not done + rm -f "$aupfile" + else + log "AUIP file '$aupfile' is not really a file, weird" + fi upstream_uip=true ;; esac -- cgit v1.2.3