diff options
-rwxr-xr-x | bin/ftpsync | 411 | ||||
-rwxr-xr-x | bin/pushpdo | 58 | ||||
-rwxr-xr-x | bin/runmirrors | 94 | ||||
-rwxr-xr-x | bin/websync | 176 |
4 files changed, 369 insertions, 370 deletions
diff --git a/bin/ftpsync b/bin/ftpsync index f70d506..6e9daeb 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -68,44 +68,44 @@ VERSION="80286" # "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the # trick. check_commandline() { - while [ $# -gt 0 ]; do - case "$1" in - sync:stage1) - SYNCSTAGE1="true" - SYNCALL="false" - ;; - sync:stage2) - SYNCSTAGE2="true" - SYNCALL="false" - ;; - sync:callback) - SYNCCALLBACK="true" - ;; - sync:archive:*) - ARCHIVE=${1##sync:archive:} - # We do not like / or . in the remotely supplied archive name. - ARCHIVE=${ARCHIVE//\/} - ARCHIVE=${ARCHIVE//.} - ;; - sync:all) - SYNCALL="true" - ;; - sync:mhop) - SYNCMHOP="true" - ;; - *) - echo "Unknown option ${1} ignored" - ;; - esac - shift # Check next set of parameters. - done + while [ $# -gt 0 ]; do + case "$1" in + sync:stage1) + SYNCSTAGE1="true" + SYNCALL="false" + ;; + sync:stage2) + SYNCSTAGE2="true" + SYNCALL="false" + ;; + sync:callback) + SYNCCALLBACK="true" + ;; + sync:archive:*) + ARCHIVE=${1##sync:archive:} + # We do not like / or . in the remotely supplied archive name. + ARCHIVE=${ARCHIVE//\/} + ARCHIVE=${ARCHIVE//.} + ;; + sync:all) + SYNCALL="true" + ;; + sync:mhop) + SYNCMHOP="true" + ;; + *) + echo "Unknown option ${1} ignored" + ;; + esac + shift # Check next set of parameters. + done } # All the stuff we want to do when we exit, no matter where cleanup() { - trap - ERR TERM HUP INT QUIT EXIT - # all done. Mail the log, exit. - log "Mirrorsync done"; + trap - ERR TERM HUP INT QUIT EXIT + # all done. Mail the log, exit. + log "Mirrorsync done"; # Lets get a statistical value SPEED=$( @@ -121,47 +121,46 @@ cleanup() { fi log "Rsync transfer speed: ${SPEED} KB/s" - if [ -n "${MAILTO}" ]; then + if [ -n "${MAILTO}" ]; then # In case rsync had something on stderr - if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then - mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" - fi - if [ "x${ERRORSONLY}x" = "xfalsex" ]; then - # And the normal log - MAILFILES="${LOG}" - if [ "x${FULLLOGS}x" = "xtruex" ]; then - # Someone wants full logs including rsync - MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" - fi - cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} - fi - fi - - savelog "${LOGDIR}/rsync-${NAME}.log" - savelog "${LOGDIR}/rsync-${NAME}.error" - savelog "$LOG" > /dev/null - - rm -f "${LOCK}" + if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then + mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" + fi + if [ "x${ERRORSONLY}x" = "xfalsex" ]; then + # And the normal log + MAILFILES="${LOG}" + if [ "x${FULLLOGS}x" = "xtruex" ]; then + # Someone wants full logs including rsync + MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" + fi + cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} + fi + fi + + savelog "${LOGDIR}/rsync-${NAME}.log" + savelog "${LOGDIR}/rsync-${NAME}.error" + savelog "$LOG" > /dev/null + + rm -f "${LOCK}" } # Check rsyncs return value check_rsync() { - - ret=$1 - msg=$2 - - # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED - # and us re-running. If it's not, uplink is broken anyways. - case "${ret}" in - 0) return 0;; - 24) return 0;; - 23) return 2;; - 30) return 2;; - *) - error "ERROR: ${msg}" - return 1 - ;; - esac + ret=$1 + msg=$2 + + # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED + # and us re-running. If it's not, uplink is broken anyways. + case "${ret}" in + 0) return 0;; + 24) return 0;; + 23) return 2;; + 30) return 2;; + *) + error "ERROR: ${msg}" + return 1 + ;; + esac } ######################################################################## @@ -181,37 +180,37 @@ SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""} # Now, check if we got told about stuff via ssh if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then # We deliberately add "nothing" and ignore it right again, to avoid - # people from outside putting some set options in the first place, - # making us parse them... - set "nothing" "${SSH_ORIGINAL_COMMAND}" - shift - # Yes, unqouted $* here. Or the function will only see it as one - # parameter, which doesnt help the case in it. - check_commandline $* + # people from outside putting some set options in the first place, + # making us parse them... + set "nothing" "${SSH_ORIGINAL_COMMAND}" + shift + # Yes, unqouted $* here. Or the function will only see it as one + # parameter, which doesnt help the case in it. + check_commandline $* fi # Now, we can locally override all the above variables by just putting # them into the .ssh/authorized_keys file forced command. if [ -n "${ORIGINAL_COMMAND}" ]; then - set ${ORIGINAL_COMMAND} - check_commandline $* + set ${ORIGINAL_COMMAND} + check_commandline $* fi # If we have been told to do stuff for a different archive than default, # set the name accordingly. ARCHIVE=${ARCHIVE:-""} if [ -n "${ARCHIVE}" ]; then - NAME="${NAME}-${ARCHIVE}" + NAME="${NAME}-${ARCHIVE}" fi # Now source the config for the archive we run on. # (Yes, people can also overwrite the options above in the config file # if they want to) if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then - . "${BASEDIR}/etc/${NAME}.conf" + . "${BASEDIR}/etc/${NAME}.conf" else - echo "Nono, you can't tell us about random archives. Bad boy!" - exit 1 + echo "Nono, you can't tell us about random archives. Bad boy!" + exit 1 fi ######################################################################## @@ -250,11 +249,11 @@ PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} # Where to send mails about mirroring to? if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then - # We are not on a debian.org host - MAILTO=${MAILTO:-"root"} + # We are not on a debian.org host + MAILTO=${MAILTO:-"root"} else - # Yay, on a .debian.org host - MAILTO=${MAILTO:-"mirrorlogs@debian.org"} + # Yay, on a .debian.org host + MAILTO=${MAILTO:-"mirrorlogs@debian.org"} fi # Want errors only or every log? ERRORSONLY=${ERRORSONLY:-"true"} @@ -330,10 +329,10 @@ ARCH_EXCLUDE=${ARCH_EXCLUDE:-""} for ARCH in ${ARCH_EXCLUDE}; do EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude disks-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes" if [ "${ARCH}" = "source" ]; then - if [ -z ${SOURCE_EXCLUDE} ]; then - SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc " - fi - fi + if [ -z ${SOURCE_EXCLUDE} ]; then + SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc " + fi + fi done # Hooks @@ -370,29 +369,29 @@ function mainroutine() { log "Mirrorsync start" PUSHFROM="${SSH_CONNECTION%%\ *}" if [ -n "${PUSHFROM}" ]; then - log "We got pushed from ${PUSHFROM}" + log "We got pushed from ${PUSHFROM}" fi log "Acquired main lock" if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then - if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then - SYNCCALLBACK="false" - error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" - fi + if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then + SYNCCALLBACK="false" + error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" + fi fi HOOK=( - HOOKNR=1 - HOOKSCR=${HOOK1} + HOOKNR=1 + HOOKSCR=${HOOK1} ) hook $HOOK # Now, we might want to sync from anonymous too. # This is that deep in this script so hook1 could, if wanted, change things! if [ -z ${RSYNC_USER} ]; then - RSYNCPTH="${RSYNC_HOST}" + RSYNCPTH="${RSYNC_HOST}" else - RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" + RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" fi # Now do the actual mirroring, and run as long as we have an updaterequired file. @@ -400,137 +399,137 @@ function mainroutine() { export RSYNC_PROXY while [ -e "${UPDATEREQUIRED}" ]; do - log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" + log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" # if we want stage1 *or* all - if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - while [ -e "${UPDATEREQUIRED}" ]; do - rm -f "${UPDATEREQUIRED}" - log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - - set +e - # Step one, sync everything except Packages/Releases - ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" - result=$? - set -e - - log "Back from rsync with returncode ${result}" - done - else - # Fake a good resultcode - result=0 - fi # Sync stage 1? - rm -f "${UPDATEREQUIRED}" - - set +e - check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" - GO=$? - set -e - if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then - log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" - elif [ ${GO} -ne 0 ]; then - exit 3 - fi - - HOOK=( - HOOKNR=2 - HOOKSCR=${HOOK2} - ) - hook $HOOK - - # if we want stage2 *or* all - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - - set +e - # We are lucky, it worked. Now do step 2 and sync again, this time including - # the packages/releases files - ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error - result=$? - set -e - - log "Back from rsync with returncode ${result}" - else - # Fake a good resultcode - result=0 - fi # Sync stage 2? - - set +e - check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" - GO=$? - set -e - if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then - log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" - elif [ ${GO} -ne 0 ]; then - exit 4 - fi - - HOOK=( - HOOKNR=3 - HOOKSCR=${HOOK3} - ) - hook $HOOK + if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + while [ -e "${UPDATEREQUIRED}" ]; do + rm -f "${UPDATEREQUIRED}" + log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + + set +e + # Step one, sync everything except Packages/Releases + ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \ + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" + result=$? + set -e + + log "Back from rsync with returncode ${result}" + done + else + # Fake a good resultcode + result=0 + fi # Sync stage 1? + rm -f "${UPDATEREQUIRED}" + + set +e + check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" + GO=$? + set -e + if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then + log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" + elif [ ${GO} -ne 0 ]; then + exit 3 + fi + + HOOK=( + HOOKNR=2 + HOOKSCR=${HOOK2} + ) + hook $HOOK + + # if we want stage2 *or* all + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + + set +e + # We are lucky, it worked. Now do step 2 and sync again, this time including + # the packages/releases files + ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \ + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error + result=$? + set -e + + log "Back from rsync with returncode ${result}" + else + # Fake a good resultcode + result=0 + fi # Sync stage 2? + + set +e + check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" + GO=$? + set -e + if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then + log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" + elif [ ${GO} -ne 0 ]; then + exit 4 + fi + + HOOK=( + HOOKNR=3 + HOOKSCR=${HOOK3} + ) + hook $HOOK done # We only update our tracefile when we had a stage2 or an all sync. # Otherwise we would update it after stage1 already, which is wrong. - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then if [ -d "`dirname "${TO}/${TRACE}"`" ]; then LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" - echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" + echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" fi fi HOOK=( - HOOKNR=4 - HOOKSCR=${HOOK4} + HOOKNR=4 + HOOKSCR=${HOOK4} ) hook $HOOK if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then - set +e - callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" - set -e + set +e + callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" + set -e fi # Remove the Archive-Update-in-Progress file before we push our downstreams. - rm -f "${LOCK}" + rm -f "${LOCK}" if [ x${HUB} = "xtrue" ]; then - # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then + # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then RUNMIRRORARGS="" if [ -n "${ARCHIVE}" ]; then # We tell runmirrors about the archive we are running on. - RUNMIRRORARGS="-a ${ARCHIVE}" + RUNMIRRORARGS="-a ${ARCHIVE}" fi - # We also tell runmirrors that we are running it from within ftpsync, so it can change - # the way it works with mhop based on that. - RUNMIRRORARGS="${RUNMIRRORARGS} -f" - - if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then - # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will - # not get to this point. - # So if that happens, tell runmirrors we are doing mhop - RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" - elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" - elif [ "xtruex" = "x${SYNCALL}x" ]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -k all" - fi - log "Trigger slave mirrors using ${RUNMIRRORARGS}" - ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} - log "Trigger slave done" - - HOOK=( - HOOKNR=5 - HOOKSCR=${HOOK5} - ) - hook $HOOK - fi + # We also tell runmirrors that we are running it from within ftpsync, so it can change + # the way it works with mhop based on that. + RUNMIRRORARGS="${RUNMIRRORARGS} -f" + + if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then + # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will + # not get to this point. + # So if that happens, tell runmirrors we are doing mhop + RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" + elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" + elif [ "xtruex" = "x${SYNCALL}x" ]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -k all" + fi + log "Trigger slave mirrors using ${RUNMIRRORARGS}" + ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} + log "Trigger slave done" + + HOOK=( + HOOKNR=5 + HOOKSCR=${HOOK5} + ) + hook $HOOK + fi fi # All done, lets call cleanup @@ -575,7 +574,7 @@ if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then unixtime=$(date +%s) difference=$(( $unixtime - $stamptime )) if [ ${difference} -ge ${LOCKTIMEOUT} ]; then - # Took longer than 60 minutes? Assume it broke and take the lock + # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock echo "$$" > "${LOCK}" else echo "Unable to start rsync, lock file younger than one hour" diff --git a/bin/pushpdo b/bin/pushpdo index 12512b5..7dfc440 100755 --- a/bin/pushpdo +++ b/bin/pushpdo @@ -52,16 +52,16 @@ KEYFILE=${KEYFILE:-".ssh/pushpackages"} MIRRORPATH=${MIRRORPATH:-"/org/packages.debian.org/mirror/"} # where to send mails to if [ "x$(hostname -s)x" != "x${MIRRORNAME%%.debian.org}x" ]; then - # We are not on a debian.org host - MAILTO=${MAILTO:-"root"} + # We are not on a debian.org host + MAILTO=${MAILTO:-"root"} else - # Yay, on a .debian.org host - MAILTO=${MAILTO:-"mirrorlogs@debian.org"} + # Yay, on a .debian.org host + MAILTO=${MAILTO:-"mirrorlogs@debian.org"} fi if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then - error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG} - exit 5 + error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG} + exit 5 fi # Some sane defaults @@ -84,29 +84,29 @@ set +e # We use grep to easily sort out all lines having a # in front of them or are empty. egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" | while read MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do - # Process the two options that can be left blank in the config - if [ -z ${MPROTO} ]; then - MPROTO=2 - fi - if [ -z ${MKEYFILE} ]; then - MKEYFILE="${BASEDIR}/${KEYFILE}" - fi - # Now, people can do stupid things and leave out the protocol, but - # define a keyfile... - if [ ${MPROTO} -ne 1 ] && [ ${MPROTO} -ne 2 ]; then - error "Need a correct ssh protocol version for ${MLNAME}, skipping" >> ${LOG} - continue - fi - - # And finally, push the mirror - log "Pushing ${MLNAME}" >> ${LOG} - # This needs a limited ssh key on the other side, something like - # no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="rsync --server -vlogDtpr . /srv/mirrors/packages.debian.org/",from="87.106.64.223,2001:8d8:80:11::35d,powell.debian.org" ssh-rsa..... - rsync -e "ssh -i ${MKEYFILE} -${MPROTO}" -av --stats "${MIRRORPATH}" ${MUSER}@${MHOSTNAME}:/does/not/matter >"${LOGDIR}/${MLNAME}.log" - log "Pushing ${MLNAME} done" >> ${LOG} - savelog ${LOGDIR}${MLNAME}.log - - set +e + # Process the two options that can be left blank in the config + if [ -z ${MPROTO} ]; then + MPROTO=2 + fi + if [ -z ${MKEYFILE} ]; then + MKEYFILE="${BASEDIR}/${KEYFILE}" + fi + # Now, people can do stupid things and leave out the protocol, but + # define a keyfile... + if [ ${MPROTO} -ne 1 ] && [ ${MPROTO} -ne 2 ]; then + error "Need a correct ssh protocol version for ${MLNAME}, skipping" >> ${LOG} + continue + fi + + # And finally, push the mirror + log "Pushing ${MLNAME}" >> ${LOG} + # This needs a limited ssh key on the other side, something like + # no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="rsync --server -vlogDtpr . /srv/mirrors/packages.debian.org/",from="87.106.64.223,2001:8d8:80:11::35d,powell.debian.org" ssh-rsa..... + rsync -e "ssh -i ${MKEYFILE} -${MPROTO}" -av --stats "${MIRRORPATH}" ${MUSER}@${MHOSTNAME}:/does/not/matter >"${LOGDIR}/${MLNAME}.log" + log "Pushing ${MLNAME} done" >> ${LOG} + savelog ${LOGDIR}${MLNAME}.log + + set +e done exit 0 diff --git a/bin/runmirrors b/bin/runmirrors index 5b04ecb..1b4436b 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -69,7 +69,7 @@ if [ $# -gt 0 ]; then ;; *) echo "Invalid usage" - echo -e $HELP + echo -e $HELP exit 1 ;; esac @@ -119,16 +119,16 @@ PUSHDELAY=${PUSHDELAY:-600} KEYFILE=${KEYFILE:-".ssh/pushmirror"} # where to send mails to if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then - # We are not on a debian.org host - MAILTO=${MAILTO:-"root"} + # We are not on a debian.org host + MAILTO=${MAILTO:-"root"} else - # Yay, on a .debian.org host - MAILTO=${MAILTO:-"mirrorlogs@debian.org"} + # Yay, on a .debian.org host + MAILTO=${MAILTO:-"mirrorlogs@debian.org"} fi if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then - error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG} - exit 5 + error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG} + exit 5 fi # Hooks @@ -151,8 +151,8 @@ trap 'log "Mirrorpush done" >> ${LOG}; savelog "${LOG}" > /dev/null' EXIT log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> ${LOG} HOOK=( - HOOKNR=1 - HOOKSCR=${HOOK1} + HOOKNR=1 + HOOKSCR=${HOOK1} ) hook $HOOK @@ -171,15 +171,15 @@ rm -f "${LOCKDIR}/all_stage1" # We use grep to easily sort out all lines having a # in front of them or are empty. egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" | while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do - if [ "x${MTYPE}x" = "xDELAYx" ]; then - # We should wait a bit. - if [ -z ${MLNAME} ]; then - MLNAME=600 - fi - log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}" - sleep ${MLNAME} - continue - fi + if [ "x${MTYPE}x" = "xDELAYx" ]; then + # We should wait a bit. + if [ -z ${MLNAME} ]; then + MLNAME=600 + fi + log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}" + sleep ${MLNAME} + continue + fi # If we are told we have a mhop sync to do and are called from within ftpsync, # we will only look at staged/mhop entries and ignore the rest. @@ -223,36 +223,36 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do fi else MPROTO=2 - MKEYFILE="${BASEDIR}/${KEYFILE}" + MKEYFILE="${BASEDIR}/${KEYFILE}" SSHOPT="" fi - # Built our array - SIGNAL_OPTS=( - MIRROR="${MLNAME}" - HOSTNAME="${MHOSTNAME}" - USERNAME="${MUSER}" - SSHPROTO="${MPROTO}" - SSHKEY="${MKEYFILE}" - SSHOPTS="${SSHOPT/ /#}" - PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1" - PUSHTYPE="${MTYPE}" - PUSHARCHIVE=${PUSHARCHIVE} - PUSHKIND=${PUSHKIND} - FROMFTPSYNC=${FROMFTPSYNC} - ) - - # And finally, push the mirror - log "Trigger ${MLNAME}" >> ${LOG} - signal "${SIGNAL_OPTS}" & - log "Trigger for ${MLNAME} done" >> ${LOG} - - HOOK=( - HOOKNR=2 - HOOKSCR=${HOOK2} - ) - hook $HOOK - set +e + # Built our array + SIGNAL_OPTS=( + MIRROR="${MLNAME}" + HOSTNAME="${MHOSTNAME}" + USERNAME="${MUSER}" + SSHPROTO="${MPROTO}" + SSHKEY="${MKEYFILE}" + SSHOPTS="${SSHOPT/ /#}" + PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1" + PUSHTYPE="${MTYPE}" + PUSHARCHIVE=${PUSHARCHIVE} + PUSHKIND=${PUSHKIND} + FROMFTPSYNC=${FROMFTPSYNC} + ) + + # And finally, push the mirror + log "Trigger ${MLNAME}" >> ${LOG} + signal "${SIGNAL_OPTS}" & + log "Trigger for ${MLNAME} done" >> ${LOG} + + HOOK=( + HOOKNR=2 + HOOKSCR=${HOOK2} + ) + hook $HOOK + set +e done # If we are run from within ftpsync *and* have an mhop push to send on, we have @@ -278,8 +278,8 @@ if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then fi HOOK=( - HOOKNR=3 - HOOKSCR=${HOOK3} + HOOKNR=3 + HOOKSCR=${HOOK3} ) hook $HOOK diff --git a/bin/websync b/bin/websync index bcb2e52..b96ba54 100755 --- a/bin/websync +++ b/bin/websync @@ -48,51 +48,51 @@ VERSION="0815" ######################################################################## # All the stuff we want to do when we exit, no matter where cleanup() { - trap - ERR TERM HUP INT QUIT EXIT - # all done. Mail the log, exit. - log "Mirrorsync done"; - if [ -n "${MAILTO}" ]; then + trap - ERR TERM HUP INT QUIT EXIT + # all done. Mail the log, exit. + log "Mirrorsync done"; + if [ -n "${MAILTO}" ]; then # In case rsync had something on stderr - if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then - mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" - fi - if [ "x${ERRORSONLY}x" = "xfalsex" ]; then - # And the normal log - MAILFILES="${LOG}" - if [ "x${FULLLOGS}x" = "xtruex" ]; then - # Someone wants full logs including rsync - MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" - fi - cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} - fi - fi - - savelog "${LOGDIR}/rsync-${NAME}.log" - savelog "${LOGDIR}/rsync-${NAME}.error" - savelog "$LOG" > /dev/null - - rm -f "${LOCK}" + if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then + mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error" + fi + if [ "x${ERRORSONLY}x" = "xfalsex" ]; then + # And the normal log + MAILFILES="${LOG}" + if [ "x${FULLLOGS}x" = "xtruex" ]; then + # Someone wants full logs including rsync + MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log" + fi + cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} + fi + fi + + savelog "${LOGDIR}/rsync-${NAME}.log" + savelog "${LOGDIR}/rsync-${NAME}.error" + savelog "$LOG" > /dev/null + + rm -f "${LOCK}" } # Check rsyncs return value check_rsync() { - ret=$1 - msg=$2 - - # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED - # and us re-running. If it's not, uplink is broken anyways. - case "${ret}" in - 0) return 0;; - 24) return 0;; - 23) return 2;; - 30) return 2;; - *) - error "ERROR: ${msg}" - return 1 - ;; - esac + ret=$1 + msg=$2 + + # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED + # and us re-running. If it's not, uplink is broken anyways. + case "${ret}" in + 0) return 0;; + 24) return 0;; + 23) return 2;; + 30) return 2;; + *) + error "ERROR: ${msg}" + return 1 + ;; + esac } ######################################################################## @@ -127,11 +127,11 @@ PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} # Where to send mails about mirroring to? if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then - # We are not on a debian.org host - MAILTO=${MAILTO:-"root"} + # We are not on a debian.org host + MAILTO=${MAILTO:-"root"} else - # Yay, on a .debian.org host - MAILTO=${MAILTO:-"mirrorlogs@debian.org"} + # Yay, on a .debian.org host + MAILTO=${MAILTO:-"mirrorlogs@debian.org"} fi # Want errors only or every log? ERRORSONLY=${ERRORSONLY:-"true"} @@ -205,13 +205,13 @@ touch "${UPDATEREQUIRED}" # Check to see if another sync is in progress if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then - if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then - # Process does either not exist or is not owned by us. - echo "$$" > "${LOCK}" - else - echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})" - exit 1 - fi + if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then + # Process does either not exist or is not owned by us. + echo "$$" > "${LOCK}" + else + echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})" + exit 1 + fi fi trap cleanup EXIT ERR TERM HUP INT QUIT @@ -223,22 +223,22 @@ exec >"$LOG" 2>&1 </dev/null log "Mirrorsync start" PUSHFROM="${SSH_CONNECTION%%\ *}" if [ -n "${PUSHFROM}" ]; then - log "We got pushed from ${PUSHFROM}" + log "We got pushed from ${PUSHFROM}" fi log "Acquired main lock" HOOK=( - HOOKNR=1 - HOOKSCR=${HOOK1} + HOOKNR=1 + HOOKSCR=${HOOK1} ) hook $HOOK # Now, we might want to sync from anonymous too. # This is that deep in this script so hook1 could, if wanted, change things! if [ -z ${RSYNC_USER} ]; then - RSYNCPTH="${RSYNC_HOST}" + RSYNCPTH="${RSYNC_HOST}" else - RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" + RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}" fi # Now do the actual mirroring, and run as long as we have an updaterequired file. @@ -246,35 +246,35 @@ export RSYNC_PASSWORD export RSYNC_PROXY while [ -e "${UPDATEREQUIRED}" ]; do - log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" + log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" - rm -f "${UPDATEREQUIRED}" - log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" + rm -f "${UPDATEREQUIRED}" + log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} ${RSYNCPTH}::${RSYNC_PATH} ${TO}" - set +e - ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \ - ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" - result=$? - set -e + set +e + ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \ + ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error" + result=$? + set -e - log "Back from rsync with returncode ${result}" + log "Back from rsync with returncode ${result}" - set +e - check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}" - GO=$? - set -e + set +e + check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}" + GO=$? + set -e - if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then - log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" - elif [ ${GO} -ne 0 ]; then - exit 3 - fi + if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then + log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" + elif [ ${GO} -ne 0 ]; then + exit 3 + fi - HOOK=( - HOOKNR=2 - HOOKSCR=${HOOK2} - ) - hook $HOOK + HOOK=( + HOOKNR=2 + HOOKSCR=${HOOK2} + ) + hook $HOOK done @@ -284,21 +284,21 @@ echo "Used websync version: ${VERSION}" >> "${TO}/${TRACE}" echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" HOOK=( - HOOKNR=3 - HOOKSCR=${HOOK3} + HOOKNR=3 + HOOKSCR=${HOOK3} ) hook $HOOK if [ x${HUB} = "xtrue" ]; then - log "Trigger slave mirrors" - ${BASEDIR}/bin/runmirrors "websync" - log "Trigger slave done" - - HOOK=( - HOOKNR=4 - HOOKSCR=${HOOK4} - ) - hook $HOOK + log "Trigger slave mirrors" + ${BASEDIR}/bin/runmirrors "websync" + log "Trigger slave done" + + HOOK=( + HOOKNR=4 + HOOKSCR=${HOOK4} + ) + hook $HOOK fi # All done, rest is done by cleanup hook. |