diff options
Diffstat (limited to 'bin')
-rwxr-xr-x | bin/ftpsync | 306 |
1 files changed, 165 insertions, 141 deletions
diff --git a/bin/ftpsync b/bin/ftpsync index c6ee505..c9543f8 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -316,6 +316,157 @@ HOOK5=${HOOK5:-""} # Are we a hub? HUB=${HUB:-"false"} +######################################################################## +# Really nothing to see below here. Only code follows. # +######################################################################## +# A function for the majority of the work. This function *CAN* run backgrounded. +# It will be when we simply sync all. Staged syncs will not background, of course. +function mainroutine() { +# Look who pushed us and note that in the log. + log "Mirrorsync start" + PUSHFROM="${SSH_CONNECTION%%\ *}" + if [ -n "${PUSHFROM}" ]; then + log "We got pushed from ${PUSHFROM}" + fi + log "Acquired main lock" + + if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then + if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then + SYNCCALLBACK="false" + error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" + fi + fi + + HOOK=( + HOOKNR=1 + HOOKSCR=${HOOK1} + ) + hook $HOOK + + # Now, we might want to sync from anonymous too. + # This is that deep in this script so hook1 could, if wanted, change things! + if [ -z ${RSYNC_USER} ]; then + RSYNCPTH="${RSYNC_HOST}" + else + RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" + fi + + # Now do the actual mirroring, and run as long as we have an updaterequired file. + export RSYNC_PASSWORD + export RSYNC_PROXY + + while [ -e "${UPDATEREQUIRED}" ]; do + log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" + + # if we want stage1 *or* all + if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + while [ -e "${UPDATEREQUIRED}" ]; do + rm -f "${UPDATEREQUIRED}" + log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + + set +e + # Step one, sync everything except Packages/Releases + ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" + result=$? + set -e + + log "Back from rsync with returncode ${result}" + done + else + # Fake a good resultcode + result=0 + fi # Sync stage 1? + rm -f "${UPDATEREQUIRED}" + + set +e + check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" + GO=$? + set -e + if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then + log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" + elif [ ${GO} -ne 0 ]; then + exit 3 + fi + + HOOK=( + HOOKNR=2 + HOOKSCR=${HOOK2} + ) + hook $HOOK + + # if we want stage2 *or* all + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + + set +e + # We are lucky, it worked. Now do step 2 and sync again, this time including + # the packages/releases files + ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error + result=$? + set -e + + log "Back from rsync with returncode ${result}" + else + # Fake a good resultcode + result=0 + fi # Sync stage 2? + + set +e + check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" + GO=$? + set -e + if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then + log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" + elif [ ${GO} -ne 0 ]; then + exit 4 + fi + + HOOK=( + HOOKNR=3 + HOOKSCR=${HOOK3} + ) + hook $HOOK + done + + if [ -d "`dirname "${TO}/${TRACE}"`" ]; then + LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" + echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" + echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" + fi + + HOOK=( + HOOKNR=4 + HOOKSCR=${HOOK4} + ) + hook $HOOK + + if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then + set +e + callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" + set -e + fi + + if [ x${HUB} = "xtrue" ]; then + # Only trigger slave mirrors if we had a push for stage2 or all + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + log "Trigger slave mirrors" + ${BASEDIR}/bin/runmirrors "${ARCHIVE}" + log "Trigger slave done" + + HOOK=( + HOOKNR=5 + HOOKSCR=${HOOK5} + ) + hook $HOOK + fi + fi + # All done, lets call cleanup + cleanup +} +######################################################################## + # Some sane defaults cd "${BASEDIR}" umask 022 @@ -344,151 +495,24 @@ if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then fi fi -# No matter how we exit this script, we always want to run the cleanup action -trap cleanup EXIT ERR TERM HUP INT QUIT +# When we exit normally we call cleanup on our own. Otherwise we want it called by +# this trap. (We can not trap on EXIT, because that is called when the main script +# exits. Which also happens when we background the mainroutine, ie. while we still +# run!) +trap cleanup ERR TERM HUP INT QUIT # Start log by redirecting everything there. exec >"$LOG" 2>&1 </dev/null -# Look who pushed us and note that in the log. -log "Mirrorsync start" -PUSHFROM="${SSH_CONNECTION%%\ *}" -if [ -n "${PUSHFROM}" ]; then - log "We got pushed from ${PUSHFROM}" -fi -log "Acquired main lock" +# All the rest of the action can be backgrounded (when we do not need to wait, +# like for staged pushed). So have a function and call it here, depending on +# the way we are called -if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then - if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then - SYNCCALLBACK="false" - error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" - fi -fi - -HOOK=( - HOOKNR=1 - HOOKSCR=${HOOK1} -) -hook $HOOK - -# Now, we might want to sync from anonymous too. -# This is that deep in this script so hook1 could, if wanted, change things! -if [ -z ${RSYNC_USER} ]; then - RSYNCPTH="${RSYNC_HOST}" +if [ "xtruex" = "x${SYNCALL}x" ]; then + # We have a simple normal mirror run, not staged. Lets background it. + mainroutine & else - RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" + # Staged run, we have to wait until we are done before we exit, uplink + # is using that for synchronisation + mainroutine fi - -# Now do the actual mirroring, and run as long as we have an updaterequired file. -export RSYNC_PASSWORD -export RSYNC_PROXY - -while [ -e "${UPDATEREQUIRED}" ]; do - log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" - - # if we want stage1 *or* all - if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - while [ -e "${UPDATEREQUIRED}" ]; do - rm -f "${UPDATEREQUIRED}" - log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - - set +e - # Step one, sync everything except Packages/Releases - ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" - result=$? - set -e - - log "Back from rsync with returncode ${result}" - done - else - # Fake a good resultcode - result=0 - fi # Sync stage 1? - rm -f "${UPDATEREQUIRED}" - - set +e - check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" - GO=$? - set -e - if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then - log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" - elif [ ${GO} -ne 0 ]; then - exit 3 - fi - - HOOK=( - HOOKNR=2 - HOOKSCR=${HOOK2} - ) - hook $HOOK - - # if we want stage2 *or* all - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - - set +e - # We are lucky, it worked. Now do step 2 and sync again, this time including - # the packages/releases files - ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error - result=$? - set -e - - log "Back from rsync with returncode ${result}" - else - # Fake a good resultcode - result=0 - fi # Sync stage 2? - - set +e - check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" - GO=$? - set -e - if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then - log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" - elif [ ${GO} -ne 0 ]; then - exit 4 - fi - - HOOK=( - HOOKNR=3 - HOOKSCR=${HOOK3} - ) - hook $HOOK -done - -if [ -d "`dirname "${TO}/${TRACE}"`" ]; then - LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" - echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" - echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" -fi - -HOOK=( - HOOKNR=4 - HOOKSCR=${HOOK4} -) -hook $HOOK - -if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then - set +e - callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" - set -e -fi - -if [ x${HUB} = "xtrue" ]; then - # Only trigger slave mirrors if we had a push for stage2 or all - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - log "Trigger slave mirrors" - ${BASEDIR}/bin/runmirrors "${ARCHIVE}" - log "Trigger slave done" - - HOOK=( - HOOKNR=5 - HOOKSCR=${HOOK5} - ) - hook $HOOK - fi -fi - -# All done, rest is done by cleanup hook. |