From 5d9719e346af8580cb0e63227e83413fa2fa0f56 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Sat, 2 Jan 2010 22:10:26 +0100 Subject: space, lotsa space change all the indentation stuff to match one schema. 4 spaces indent, no tabs. was pretty mixed up. Signed-off-by: Joerg Jaspert --- bin/ftpsync | 411 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 205 insertions(+), 206 deletions(-) (limited to 'bin/ftpsync') diff --git a/bin/ftpsync b/bin/ftpsync index f70d506..6e9daeb 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -68,44 +68,44 @@ VERSION="80286" # "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the # trick. check_commandline() { - while [ $# -gt 0 ]; do - case "$1" in - sync:stage1) - SYNCSTAGE1="true" - SYNCALL="false" - ;; - sync:stage2) - SYNCSTAGE2="true" - SYNCALL="false" - ;; - sync:callback) - SYNCCALLBACK="true" - ;; - sync:archive:*) - ARCHIVE=${1##sync:archive:} - # We do not like / or . in the remotely supplied archive name. - ARCHIVE=${ARCHIVE//\/} - ARCHIVE=${ARCHIVE//.} - ;; - sync:all) - SYNCALL="true" - ;; - sync:mhop) - SYNCMHOP="true" - ;; - *) - echo "Unknown option ${1} ignored" - ;; - esac - shift # Check next set of parameters. - done + while [ $# -gt 0 ]; do + case "$1" in + sync:stage1) + SYNCSTAGE1="true" + SYNCALL="false" + ;; + sync:stage2) + SYNCSTAGE2="true" + SYNCALL="false" + ;; + sync:callback) + SYNCCALLBACK="true" + ;; + sync:archive:*) + ARCHIVE=${1##sync:archive:} + # We do not like / or . in the remotely supplied archive name. + ARCHIVE=${ARCHIVE//\/} + ARCHIVE=${ARCHIVE//.} + ;; + sync:all) + SYNCALL="true" + ;; + sync:mhop) + SYNCMHOP="true" + ;; + *) + echo "Unknown option ${1} ignored" + ;; + esac + shift # Check next set of parameters. + done } # All the stuff we want to do when we exit, no matter where cleanup() { - trap - ERR TERM HUP INT QUIT EXIT - # all done. Mail the log, exit. - log "Mirrorsync done"; + trap - ERR TERM HUP INT QUIT EXIT + # all done. Mail the log, exit. + log "Mirrorsync done"; # Lets get a statistical value SPEED=$( @@ -121,47 +121,46 @@ cleanup() { fi log "Rsync transfer speed: ${SPEED} KB/s" - if [ -n "${MAILTO}" ]; then + if [ -n "${MAILTO}" ]; then # In case rsync had something on stderr - if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then - mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" - fi - if [ "x${ERRORSONLY}x" = "xfalsex" ]; then - # And the normal log - MAILFILES="${LOG}" - if [ "x${FULLLOGS}x" = "xtruex" ]; then - # Someone wants full logs including rsync - MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" - fi - cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} - fi - fi - - savelog "${LOGDIR}/rsync-${NAME}.log" - savelog "${LOGDIR}/rsync-${NAME}.error" - savelog "$LOG" > /dev/null - - rm -f "${LOCK}" + if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then + mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" + fi + if [ "x${ERRORSONLY}x" = "xfalsex" ]; then + # And the normal log + MAILFILES="${LOG}" + if [ "x${FULLLOGS}x" = "xtruex" ]; then + # Someone wants full logs including rsync + MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" + fi + cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} + fi + fi + + savelog "${LOGDIR}/rsync-${NAME}.log" + savelog "${LOGDIR}/rsync-${NAME}.error" + savelog "$LOG" > /dev/null + + rm -f "${LOCK}" } # Check rsyncs return value check_rsync() { - - ret=$1 - msg=$2 - - # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED - # and us re-running. If it's not, uplink is broken anyways. - case "${ret}" in - 0) return 0;; - 24) return 0;; - 23) return 2;; - 30) return 2;; - *) - error "ERROR: ${msg}" - return 1 - ;; - esac + ret=$1 + msg=$2 + + # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED + # and us re-running. If it's not, uplink is broken anyways. + case "${ret}" in + 0) return 0;; + 24) return 0;; + 23) return 2;; + 30) return 2;; + *) + error "ERROR: ${msg}" + return 1 + ;; + esac } ######################################################################## @@ -181,37 +180,37 @@ SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""} # Now, check if we got told about stuff via ssh if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then # We deliberately add "nothing" and ignore it right again, to avoid - # people from outside putting some set options in the first place, - # making us parse them... - set "nothing" "${SSH_ORIGINAL_COMMAND}" - shift - # Yes, unqouted $* here. Or the function will only see it as one - # parameter, which doesnt help the case in it. - check_commandline $* + # people from outside putting some set options in the first place, + # making us parse them... + set "nothing" "${SSH_ORIGINAL_COMMAND}" + shift + # Yes, unqouted $* here. Or the function will only see it as one + # parameter, which doesnt help the case in it. + check_commandline $* fi # Now, we can locally override all the above variables by just putting # them into the .ssh/authorized_keys file forced command. if [ -n "${ORIGINAL_COMMAND}" ]; then - set ${ORIGINAL_COMMAND} - check_commandline $* + set ${ORIGINAL_COMMAND} + check_commandline $* fi # If we have been told to do stuff for a different archive than default, # set the name accordingly. ARCHIVE=${ARCHIVE:-""} if [ -n "${ARCHIVE}" ]; then - NAME="${NAME}-${ARCHIVE}" + NAME="${NAME}-${ARCHIVE}" fi # Now source the config for the archive we run on. # (Yes, people can also overwrite the options above in the config file # if they want to) if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then - . "${BASEDIR}/etc/${NAME}.conf" + . "${BASEDIR}/etc/${NAME}.conf" else - echo "Nono, you can't tell us about random archives. Bad boy!" - exit 1 + echo "Nono, you can't tell us about random archives. Bad boy!" + exit 1 fi ######################################################################## @@ -250,11 +249,11 @@ PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} # Where to send mails about mirroring to? if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then - # We are not on a debian.org host - MAILTO=${MAILTO:-"root"} + # We are not on a debian.org host + MAILTO=${MAILTO:-"root"} else - # Yay, on a .debian.org host - MAILTO=${MAILTO:-"mirrorlogs@debian.org"} + # Yay, on a .debian.org host + MAILTO=${MAILTO:-"mirrorlogs@debian.org"} fi # Want errors only or every log? ERRORSONLY=${ERRORSONLY:-"true"} @@ -330,10 +329,10 @@ ARCH_EXCLUDE=${ARCH_EXCLUDE:-""} for ARCH in ${ARCH_EXCLUDE}; do EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude disks-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes" if [ "${ARCH}" = "source" ]; then - if [ -z ${SOURCE_EXCLUDE} ]; then - SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc " - fi - fi + if [ -z ${SOURCE_EXCLUDE} ]; then + SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc " + fi + fi done # Hooks @@ -370,29 +369,29 @@ function mainroutine() { log "Mirrorsync start" PUSHFROM="${SSH_CONNECTION%%\ *}" if [ -n "${PUSHFROM}" ]; then - log "We got pushed from ${PUSHFROM}" + log "We got pushed from ${PUSHFROM}" fi log "Acquired main lock" if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then - if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then - SYNCCALLBACK="false" - error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" - fi + if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then + SYNCCALLBACK="false" + error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" + fi fi HOOK=( - HOOKNR=1 - HOOKSCR=${HOOK1} + HOOKNR=1 + HOOKSCR=${HOOK1} ) hook $HOOK # Now, we might want to sync from anonymous too. # This is that deep in this script so hook1 could, if wanted, change things! if [ -z ${RSYNC_USER} ]; then - RSYNCPTH="${RSYNC_HOST}" + RSYNCPTH="${RSYNC_HOST}" else - RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" + RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" fi # Now do the actual mirroring, and run as long as we have an updaterequired file. @@ -400,137 +399,137 @@ function mainroutine() { export RSYNC_PROXY while [ -e "${UPDATEREQUIRED}" ]; do - log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" + log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" # if we want stage1 *or* all - if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - while [ -e "${UPDATEREQUIRED}" ]; do - rm -f "${UPDATEREQUIRED}" - log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - - set +e - # Step one, sync everything except Packages/Releases - ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" - result=$? - set -e - - log "Back from rsync with returncode ${result}" - done - else - # Fake a good resultcode - result=0 - fi # Sync stage 1? - rm -f "${UPDATEREQUIRED}" - - set +e - check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" - GO=$? - set -e - if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then - log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" - elif [ ${GO} -ne 0 ]; then - exit 3 - fi - - HOOK=( - HOOKNR=2 - HOOKSCR=${HOOK2} - ) - hook $HOOK - - # if we want stage2 *or* all - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - - set +e - # We are lucky, it worked. Now do step 2 and sync again, this time including - # the packages/releases files - ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error - result=$? - set -e - - log "Back from rsync with returncode ${result}" - else - # Fake a good resultcode - result=0 - fi # Sync stage 2? - - set +e - check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" - GO=$? - set -e - if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then - log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" - elif [ ${GO} -ne 0 ]; then - exit 4 - fi - - HOOK=( - HOOKNR=3 - HOOKSCR=${HOOK3} - ) - hook $HOOK + if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + while [ -e "${UPDATEREQUIRED}" ]; do + rm -f "${UPDATEREQUIRED}" + log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + + set +e + # Step one, sync everything except Packages/Releases + ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" + result=$? + set -e + + log "Back from rsync with returncode ${result}" + done + else + # Fake a good resultcode + result=0 + fi # Sync stage 1? + rm -f "${UPDATEREQUIRED}" + + set +e + check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" + GO=$? + set -e + if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then + log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" + elif [ ${GO} -ne 0 ]; then + exit 3 + fi + + HOOK=( + HOOKNR=2 + HOOKSCR=${HOOK2} + ) + hook $HOOK + + # if we want stage2 *or* all + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + + set +e + # We are lucky, it worked. Now do step 2 and sync again, this time including + # the packages/releases files + ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error + result=$? + set -e + + log "Back from rsync with returncode ${result}" + else + # Fake a good resultcode + result=0 + fi # Sync stage 2? + + set +e + check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" + GO=$? + set -e + if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then + log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" + elif [ ${GO} -ne 0 ]; then + exit 4 + fi + + HOOK=( + HOOKNR=3 + HOOKSCR=${HOOK3} + ) + hook $HOOK done # We only update our tracefile when we had a stage2 or an all sync. # Otherwise we would update it after stage1 already, which is wrong. - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then if [ -d "`dirname "${TO}/${TRACE}"`" ]; then LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" - echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" + echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" fi fi HOOK=( - HOOKNR=4 - HOOKSCR=${HOOK4} + HOOKNR=4 + HOOKSCR=${HOOK4} ) hook $HOOK if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then - set +e - callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" - set -e + set +e + callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" + set -e fi # Remove the Archive-Update-in-Progress file before we push our downstreams. - rm -f "${LOCK}" + rm -f "${LOCK}" if [ x${HUB} = "xtrue" ]; then - # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then + # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then RUNMIRRORARGS="" if [ -n "${ARCHIVE}" ]; then # We tell runmirrors about the archive we are running on. - RUNMIRRORARGS="-a ${ARCHIVE}" + RUNMIRRORARGS="-a ${ARCHIVE}" fi - # We also tell runmirrors that we are running it from within ftpsync, so it can change - # the way it works with mhop based on that. - RUNMIRRORARGS="${RUNMIRRORARGS} -f" - - if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then - # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will - # not get to this point. - # So if that happens, tell runmirrors we are doing mhop - RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" - elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" - elif [ "xtruex" = "x${SYNCALL}x" ]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -k all" - fi - log "Trigger slave mirrors using ${RUNMIRRORARGS}" - ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} - log "Trigger slave done" - - HOOK=( - HOOKNR=5 - HOOKSCR=${HOOK5} - ) - hook $HOOK - fi + # We also tell runmirrors that we are running it from within ftpsync, so it can change + # the way it works with mhop based on that. + RUNMIRRORARGS="${RUNMIRRORARGS} -f" + + if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then + # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will + # not get to this point. + # So if that happens, tell runmirrors we are doing mhop + RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" + elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" + elif [ "xtruex" = "x${SYNCALL}x" ]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -k all" + fi + log "Trigger slave mirrors using ${RUNMIRRORARGS}" + ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} + log "Trigger slave done" + + HOOK=( + HOOKNR=5 + HOOKSCR=${HOOK5} + ) + hook $HOOK + fi fi # All done, lets call cleanup @@ -575,7 +574,7 @@ if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then unixtime=$(date +%s) difference=$(( $unixtime - $stamptime )) if [ ${difference} -ge ${LOCKTIMEOUT} ]; then - # Took longer than 60 minutes? Assume it broke and take the lock + # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock echo "$$" > "${LOCK}" else echo "Unable to start rsync, lock file younger than one hour" -- cgit v1.2.3