diff options
Diffstat (limited to 'etc')
| -rw-r--r-- | etc/common | 109 | ||||
| -rw-r--r-- | etc/runmirrors.mirror.sample | 32 |
2 files changed, 91 insertions, 50 deletions
@@ -17,6 +17,8 @@ # then push stage2 # $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) # $PUSHCB - do we want a callback? +# $PUSHKIND - whats going on? are we doing mhop push or already stage2? +# $FROMFTPSYNC - set to true if we run from within ftpsync. # # This function assumes that the variable LOG is set to a directory where # logfiles can be written to. @@ -38,6 +40,8 @@ signal () { PUSHTYPE=${PUSHTYPE:-"all"} PUSHARCHIVE=${PUSHARCHIVE:-""} PUSHCB=${PUSHCB:-""} + PUSHKIND=${PUSHKIND:-"all"} + FROMFTPSYNC=${FROMFTPSYNC:-"false"} # And now get # back to space... SSHOPTS=${SSHOPTS/\#/ } @@ -45,23 +49,26 @@ signal () { # Defaults we always want, no matter what SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + # If there are userdefined ssh options, add them. if [ -n "${SSH_OPTS}" ]; then SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}" fi + # Does this machine need a special key? if [ -n "${SSHKEY}" ]; then SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}" fi + # Does this machine have an extra own set of ssh options? if [ -n "${SSHOPTS}" ]; then SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}" fi + # Set the protocol version if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ] && [ ${SSHPROTO} -ne 99 ]; then # Idiots, we only want 1 or 2. Cant decide? Lets force 2. SSHPROTO=2 fi - if [ -n "${SSHPROTO}" ] && [ ${SSHPROTO} -ne 99 ]; then SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}" fi @@ -69,64 +76,82 @@ signal () { date -u >> "${LOGDIR}/${MIRROR}.log" PUSHARGS="" + # Archive is non-empty, so tell the downstreams if [ -n "${PUSHARCHIVE}" ]; then PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" fi + # We have a callback wish, tell downstreams if [ -n "${PUSHCB}" ]; then PUSHARGS="${PUSHARGS} sync:callback" fi + # If we are running an mhop push AND our downstream is one to receive it, tell it. + if [ "xmhopx" = "x${PUSHKIND}x" ] && [ "xmhopx" = "x${PUSHTYPE}x" ]; then + PUSHARGS="${PUSHARGS} sync:mhop" + fi if [ "xallx" = "x${PUSHTYPE}x" ]; then - # Default normal "fire and forget" push + # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log" PUSHARGS1="sync:all" ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 & - elif [ "xstagedx" = "x${PUSHTYPE}x" ]; then - # Want a staged push. Fine, lets do that + elif [ "xstagedx" = "x${PUSHTYPE}x" ] || [ "xmhopx" = "x${PUSHTYPE}x" ]; then + # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" - # Step1: Do a push to only sync stage1, do not background - PUSHARGS1="sync:stage1" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 - touch "${PUSHLOCKOWN}" - - # Step2: Wait for all the other "lock"files to appear. - tries=0 - # We do not wait forever - while [ ${tries} -lt ${PUSHDELAY} ]; do - total=0 - found=0 - for file in ${PUSHLOCKS}; do - total=$((total + 1)) - if [ -f ${file} ]; then - found=$((found + 1)) + # Only send stage1 if we havent already send it. When called with stage2, we already did. + if [ "xstage2x" != "x${PUSHKIND}x" ]; then + # Step1: Do a push to only sync stage1, do not background + PUSHARGS1="sync:stage1" + ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + touch "${PUSHLOCKOWN}" + + # Step2: Wait for all the other "lock"files to appear. + tries=0 + # We do not wait forever + while [ ${tries} -lt ${PUSHDELAY} ]; do + total=0 + found=0 + for file in ${PUSHLOCKS}; do + total=$((total + 1)) + if [ -f ${file} ]; then + found=$((found + 1)) + fi + done + if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then + touch "${LOCKDIR}/all_stage1" + break fi + tries=$((tries + 5)) + sleep 5 done - if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then - touch "${LOCKDIR}/all_stage1" - break + # In case we did not have all PUSHLOCKS and still continued, note it + # This is a little racy, especially if the other parts decide to do this + # at the same time, but it wont hurt more than a mail too much, so I don't care much + if [ ${tries} -ge ${PUSHDELAY} ]; then + echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" + for file in ${PUSHLOCKS}; do + if [ ! -f ${file} ]; then + echo "${file}" >> "${LOGDIR}/${MIRROR}.log" + error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" + fi + done fi - tries=$((tries + 5)) - sleep 5 - done - # In case we did not have all PUSHLOCKS and still continued, note it - # This is a little racy, especially if the other parts decide to do this - # at the same time, but it wont hurt more than a mail too much, so I don't care much - if [ ${tries} -ge ${PUSHDELAY} ]; then - echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" - for file in ${PUSHLOCKS}; do - if [ ! -f ${file} ]; then - echo "${file}" >> "${LOGDIR}/${MIRROR}.log" - error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" - fi - done + rm -f "${PUSHLOCKOWN}" fi - rm -f "${PUSHLOCKOWN}" - # Step3: It either timed out or we have all the "lock"files, sync stage2 - PUSHARGS2="sync:stage2" - echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + # Step3: It either timed out or we have all the "lock"files, do the rest + # If we are doing mhop AND are called from ftpsync - we now exit. + # That way we notify our uplink that we and all our clients are done with their + # stage1. It can then finish its own, and if all our upstreams downlinks are done, + # it will send us stage2. + # If we are not doing mhop or are not called from ftpsync, we start stage2 + if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then + return + else + PUSHARGS2="sync:stage2" + echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" + ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + fi else # Can't decide? Then you get nothing. return @@ -181,7 +206,7 @@ hook () { # Return the list of 2-stage mirrors. get2stage() { - egrep '^staged' "${MIRRORS}" | { + egrep '^(staged|mhop)' "${MIRRORS}" | { while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}" done diff --git a/etc/runmirrors.mirror.sample b/etc/runmirrors.mirror.sample index 2f841bb..744ad98 100644 --- a/etc/runmirrors.mirror.sample +++ b/etc/runmirrors.mirror.sample @@ -7,11 +7,15 @@ # # Type ShortName HostName User -$SOMESSHOPTION # +# The fields Type, ShortName, HostName and User are *mandantory*. # -# Type is either all or staged, meaning: +# Type is either all, staged or mhop, meaning: # all - do a "normal" push. Trigger them, go on. # staged - do a two-stage push, waiting for them after stage 2(and all others that # are staged) before doing stage2 +# mhop - send a multi-hop staged push. This will tell the mirror to initiate +# a mhop/stage1 push to its staged/mhop mirrors and then exit. +# When all mhop got back we then send the stage2 through to them. # # ShortName will be used as a shorthand in logfile outputs and for the logfile # where every ssh output gets redirected to. @@ -33,21 +37,33 @@ # want. There is currently no way around this, as that would mean # dropping backward compatibility. # +# Backwards compatibility: +# An older runmirrors script will NOT run with a newer runmirrors.mirror file, but +# a new runmirrors can run with an old runmirrors.mirror file. This should make updates +# painless. +# # Examples: # all eu.puccini puccini.debian.org archvsync 2 # -# will push puccini.debian.org, user archvsync, using ssh protocol 2 and the specified ssh key. -# +# -> will push puccini.debian.org, user archvsync, using ssh protocol 2 +# and the globally configured ssh key. # # all eu.puccini puccini.debian.org archvsync -p 2222 # -# which will do the same as above, but use port 2222 to connect to. +# -> will do the same as above, but use port 2222 to connect to. +# +# staged eu.puccini puccini.debian.org archvsync +# staged eu.powell powell.debian.org archvsync +# +# -> will push both puccini and powell in stage1, waiting for both to +# finish stage1 before stage2 gets pushed. The username will be archvsync. # -# staged eu.puccini puccini.debian.org -# staged eu.powell powell.debian.org +# staged eu.puccini puccini.debian.org archvsync +# mhop eu.powell powell.debian.org archvsync # -# will push both puccini and powell in stage1, waiting for both to -# finish stage1 before stage2 gets pushed. +# -> will do the same as above, but powell gets told about mhop and can then +# push its own staged/mhop mirrors before returning. When both returned +# then stage2 is sent to both. # # One special value is allowed: DELAY # This word has to be on a line itself, followed by a space and a number. |