diff options
| -rwxr-xr-x | bin/ftpsync | 152 | ||||
| -rwxr-xr-x | bin/runmirrors | 2 | ||||
| -rw-r--r-- | etc/common | 43 |
3 files changed, 111 insertions, 86 deletions
diff --git a/bin/ftpsync b/bin/ftpsync index d0db9a7..bcca3d9 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -30,7 +30,7 @@ set -u BASEDIR=${BASEDIR:-"${HOME}"} # Source our common functions -. ${BASEDIR}/etc/common +. "${BASEDIR}/etc/common" ######################################################################## ######################################################################## @@ -73,6 +73,9 @@ check_commandline() { ;; sync:archive:*) ARCHIVE=${1##sync:archive:} + # We do not like / or . in the remotely supplied archive name. + ARCHIVE=${ARCHIVE//\/} + ARCHIVE=${ARCHIVE//.} ;; sync:all) SYNCALL="true" @@ -93,19 +96,19 @@ cleanup() { if [ -n "${MAILTO}" ]; then # In case rsync had something on stderr if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then - cat "${LOGDIR}/rsync-${NAME}.error" | mail -e -s "$PROGRAM rsync ERROR ($(hostname -s)) [$$]" ${MAILTO} + mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" fi if [ "x${ERRORSONLY}x" = "xfalsex" ]; then # And the normal log - mail -e -s "${PROGRAM} ($(hostname)) - archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < ${LOG} + mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < ${LOG} fi fi - ${SAVELOG} ${LOGDIR}/rsync-${NAME}.log - ${SAVELOG} ${LOGDIR}/rsync-${NAME}.error + ${SAVELOG} "${LOGDIR}/rsync-${NAME}.log" + ${SAVELOG} "${LOGDIR}/rsync-${NAME}.error" ${SAVELOG} "$LOG" > /dev/null - rm -f ${LOCK}; + rm -f "${LOCK}"; } ######################################################################## @@ -122,16 +125,18 @@ if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then # We deliberately add "nothing" and ignore it right again, to avoid # people from outside putting some set options in the first place, # making us parse them... - set "nothing" ${SSH_ORIGINAL_COMMAND} + set "nothing" "${SSH_ORIGINAL_COMMAND}" shift - check_commandline "$*" + # Yes, unqouted $* here. Or the function will only see it as one + # parameter, which doesnt help the case in it. + check_commandline $* fi # Now, we can locally override all the above variables by just putting # them into the .ssh/authorized_keys file forced command. if [ -n "${ORIGINAL_COMMAND}" ]; then set ${ORIGINAL_COMMAND} - check_commandline "$*" + check_commandline $* fi # If we have been told to do stuff for a different archive than default, @@ -143,7 +148,7 @@ fi # Now source the config for the archive we run on. # (Yes, people can also overwrite the options above in the config file # if they want to) -. ${BASEDIR}/etc/${NAME}.conf +. "${BASEDIR}/etc/${NAME}.conf" ######################################################################## # Config options go here. Feel free to overwrite them in the config # @@ -179,7 +184,13 @@ TO=${TO:-"/org/ftp.debian.org/ftp/"} PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} # Where to send mails about mirroring to? -MAILTO=${MAILTO:-"mirrorlogs@debian.org"} +if [ "x${HOSTNAME}x" = "x${HOSTNAME%%.debian.org}x" ]; then + # We are not on a debian.org host + MAILTO=${MAILTO:-"root"} +else + # Yay, on a .debian.org host + MAILTO=${MAILTO:-"mirrorlogs@debian.org"} +fi # Want errors only or every log? ERRORSONLY=${ERRORSONLY:-"false"} @@ -192,7 +203,7 @@ LOCKFILE=${LOCKFILE:-"lockfile"} LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${HOSTNAME}"} # Do we need another rsync run? UPDATEREQUIRED="${TO}/Archive-Update-Required-${HOSTNAME}" -# Trace file for mirror stats and checks +# Trace file for mirror stats and checks (make sure we get full hostname) TRACE=${TRACE:-"project/trace/$(hostname -f)"} # rsync program @@ -255,12 +266,12 @@ done # Some sane defaults -cd ${BASEDIR} -umask 002 +cd "${BASEDIR}" +umask 022 # If we are here for the first time, create the # destination and the trace directory -mkdir -p ${TO}/project/trace +mkdir -p "${TO}/project/trace" # Used to make sure we will have the archive fully and completly synced before # we stop, even if we get multiple pushes while this script is running. @@ -288,12 +299,11 @@ if [ "xtruex" = "x${SYNCCALLBACK}x" ] || [ "xnonex" = "x${CALLBACKHOST}x" ] || [ error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" fi -if [ -n "${HOOK1}" ]; then - log "Running hook1: ${HOOK1}" - ${HOOK1} - result=$? - log "Back from hook1, got returncode ${result}" -fi +HOOK=( + HOOKNR=1 + HOOKSCR=${HOOK1} +) +hook $HOOK # Now, we might want to sync from anonymous too. # This is that deep in this script so hook1 could, if wanted, change things! @@ -316,7 +326,7 @@ while [ -e "${UPDATEREQUIRED}" ]; do # Step one, sync everything except Packages/Releases ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} ${TO} >${LOGDIR}/rsync-${NAME}.log 2>${LOGDIR}/rsync-${NAME}.error + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" result=$? log "Back from rsync with returncode ${result}" @@ -327,61 +337,59 @@ while [ -e "${UPDATEREQUIRED}" ]; do # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED # and us re-running. If it's not, uplink is broken anyways. - if [ $result -eq 0 ] || [ $result -eq 24 ]; then + if [ $result -ne 0 ] && [ $result -ne 24 ]; then + error "ERROR: Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}. Aborting" + exit 3 + fi - if [ -n "${HOOK2}" ]; then - log "Running hook2: ${HOOK1}" - ${HOOK2} - result=$? - log "Back from hook1, got returncode ${result}" - fi + HOOK=( + HOOKNR=2 + HOOKSCR=${HOOK2} + ) + hook $HOOK - # if we want stage2 *or* all - if [ "xtruex" = "x${SYNCSTAGE2}x"] || [ "xtruex" = "x${SYNCALL}x"]; then - log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - - # We are lucky, it worked. Now do step 2 and sync again, this time including - # the packages/releases files - ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} ${TO} >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error - result=$? - - log "Back from rsync with returncode ${result}" - else - # Fake a good resultcode - result=0 - fi # Sync stage 2? - - if [ $result -eq 0 ] || [ $result -eq 24 ]; then - if [ -n "${HOOK3}" ]; then - log "Running hook3: ${HOOK1}" - ${HOOK3} - result=$? - log "Back from hook3, got returncode ${result}" - fi - else - error "ERROR: Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" - exit 4 - fi + # if we want stage2 *or* all + if [ "xtruex" = "x${SYNCSTAGE2}x"] || [ "xtruex" = "x${SYNCALL}x"]; then + log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + + # We are lucky, it worked. Now do step 2 and sync again, this time including + # the packages/releases files + ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error + result=$? + + log "Back from rsync with returncode ${result}" else - error "ERROR: Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" - exit 3 + # Fake a good resultcode + result=0 + fi # Sync stage 2? + + if [ $result -ne 0 ] && [ $result -ne 24 ]; then + error "ERROR: Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" + exit 4 fi + + HOOK=( + HOOKNR=3 + HOOKSCR=${HOOK3} + ) + hook $HOOK done if [ -d "${TO}/project/trace" ]; then date -u > "${TO}/${TRACE}" fi -if [ -n "${HOOK4}" ]; then - log "Running hook4: ${HOOK1}" - ${HOOK4} - result=$? - log "Back from hook4, got returncode ${result}" -fi +HOOK=( + HOOKNR=4 + HOOKSCR=${HOOK4} +) +hook $HOOK if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then - callback ${CALLBACKUSER} ${CALLBACKHOST} ${CALLBACKKEY} + set +e + callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" + set -e fi if [ x${HUB} = "xtrue" ]; then @@ -389,13 +397,11 @@ if [ x${HUB} = "xtrue" ]; then ${BASEDIR}/bin/runmirrors log "Trigger slave done" - if [ -n "${HOOK5}" ]; then - log "Running hook5: ${HOOK1}" - ${HOOK5} - result=$? - log "Back from hook5, got returncode ${result}" - fi + HOOK=( + HOOKNR=5 + HOOKSCR=${HOOK5} + ) + hook $HOOK fi -# All done, remove our LOCK, rest is done by cleanup hook. -rm -f $LOCK +# All done, rest is done by cleanup hook. diff --git a/bin/runmirrors b/bin/runmirrors index 06423be..d9e7bf1 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -62,7 +62,7 @@ SSH_OPTS=${SSH_OPTS:-""} # Some sane defaults cd ${BASEDIR} -umask 002 +umask 022 # Make sure we have our log and lock directories mkdir -p ${LOGDIR} @@ -26,7 +26,7 @@ # Pushes might be done in background (for type all). signal () { ARGS="SIGNAL_OPTS[*]" - local ${!SIGNAL_OPTS} + local ${!ARGS} # Defaults we always want, no matter what SSH_OPTIONS="-o BatchMode=yes -o SetupTimeOut=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" @@ -40,11 +40,11 @@ signal () { ${SSHPROTO}=2 fi - date -u >> ${LOGDIR}/${MIRROR}.log + date -u >> "${LOGDIR}/${MIRROR}.log" if [ "xallx" = "x${PUSHTYPE}x" ]; then # Default normal "fire and forget" push - ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "sync:all" >>${LOGDIR}/${MIRROR}.log 2>&1 & + ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "sync:all" >>"${LOGDIR}/${MIRROR}.log" 2>&1 & elif [ "xstagedx" = "x{$PUSHTYPE}x"]; then # Want a staged push. Fine, lets do that @@ -58,31 +58,31 @@ signal () { # Step1: Do a push to only sync stage1, do not background PUSHARGS1="sync:stage1" - ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>${LOGDIR}/${MIRROR}.log 2>&1 - touch ${PUSHLOCKOWN} + ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + touch "${PUSHLOCKOWN}" # Step2: Wait for all the other "lock"files to appear. tries=0 # We do not wait forever - while [ $tries -lt 30 ]; do + while [ ${tries} -lt 30 ]; do total=0 found=0 for file in ${PUSHLOCKS}; do total=$((total + 1)) if [ -f ${file} ]; then - found=$(($found + 1)) + found=$((found + 1)) fi done - if [ $total -eq $found ]; + if [ ${total} -eq ${found} ]; break; fi sleep 20 done - rm -f ${PUSHLOCKOWN} + rm -f "${PUSHLOCKOWN}" - # Steo3: It either timed out or we have all the "lock"files, sync stage2 + # Step3: It either timed out or we have all the "lock"files, sync stage2 PUSHARGS1="sync:stage2" - ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>${LOGDIR}/${MIRROR}.log 2>&1 + ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 else # Can't decide? Then you get nothing. fi @@ -112,6 +112,25 @@ log () { error () { log "$@" if [ -z ${MAILTO} ]; then - echo "$@" | mail -e -s "$PROGRAM ERROR ($(hostname -s)) [$$]" ${MAILTO} + echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO} + fi +} + +# run a hook +# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR +# the script to run. +hook () { + ARGS="HOOK[*]" + local ${!ARGS} + if [ -n "${HOOKSCR}" ]; then + log "Running hook $HOOKNR: ${HOOKSCR}" + set +e + ${HOOKSCR} + result=$? + set -e + log "Back from hook $HOOKNR, got returncode ${result}" + return $result + else + return 0 fi } |