From 69fd1eda70605488c21f5773a0ba62772e14650d Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 16:52:57 +0100 Subject: common also look at mhop hosts in get2stage Signed-off-by: Joerg Jaspert --- etc/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'etc') diff --git a/etc/common b/etc/common index 465ab73..daa228d 100644 --- a/etc/common +++ b/etc/common @@ -181,7 +181,7 @@ hook () { # Return the list of 2-stage mirrors. get2stage() { - egrep '^staged' "${MIRRORS}" | { + egrep '^(staged|mhop)' "${MIRRORS}" | { while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}" done -- cgit v1.2.3 From 9aaa0aadba78e8ca6fb3fa47839a066b938ac327 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 16:54:53 +0100 Subject: Sample config make clear we need the username in the mirrors file, fix the broken example. Also mention backwards compatibility Signed-off-by: Joerg Jaspert --- etc/runmirrors.mirror.sample | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'etc') diff --git a/etc/runmirrors.mirror.sample b/etc/runmirrors.mirror.sample index 2f841bb..e2820e3 100644 --- a/etc/runmirrors.mirror.sample +++ b/etc/runmirrors.mirror.sample @@ -7,6 +7,7 @@ # # Type ShortName HostName User -$SOMESSHOPTION # +# The fields Type, ShortName, HostName and User are *mandantory*. # # Type is either all or staged, meaning: # all - do a "normal" push. Trigger them, go on. @@ -33,6 +34,11 @@ # want. There is currently no way around this, as that would mean # dropping backward compatibility. # +# Backwards compatibility: +# An older runmirrors script will NOT run with a newer runmirrors.mirror file, but +# a new runmirrors can run with an old runmirrors.mirror file. This should make updates +# painless. +# # Examples: # all eu.puccini puccini.debian.org archvsync 2 # @@ -43,11 +49,11 @@ # # which will do the same as above, but use port 2222 to connect to. # -# staged eu.puccini puccini.debian.org -# staged eu.powell powell.debian.org +# staged eu.puccini puccini.debian.org archvsync +# staged eu.powell powell.debian.org archvsync # # will push both puccini and powell in stage1, waiting for both to -# finish stage1 before stage2 gets pushed. +# finish stage1 before stage2 gets pushed. The username will be archvsync. # # One special value is allowed: DELAY # This word has to be on a line itself, followed by a space and a number. -- cgit v1.2.3 From 32d1612c28af425609306a494f476d31ad8c9d10 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:33:59 +0100 Subject: common 2 new options Signed-off-by: Joerg Jaspert --- etc/common | 2 ++ 1 file changed, 2 insertions(+) (limited to 'etc') diff --git a/etc/common b/etc/common index daa228d..78d8451 100644 --- a/etc/common +++ b/etc/common @@ -38,6 +38,8 @@ signal () { PUSHTYPE=${PUSHTYPE:-"all"} PUSHARCHIVE=${PUSHARCHIVE:-""} PUSHCB=${PUSHCB:-""} + PUSHKIND=${PUSHKIND:-"all"} + FROMFTPSYNC=${FROMFTPSYNC:-"false"} # And now get # back to space... SSHOPTS=${SSHOPTS/\#/ } -- cgit v1.2.3 From 9c0dad2d25d7abbd9842d009483e528d9a0515e4 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:34:24 +0100 Subject: common properly signal the mhop foo Signed-off-by: Joerg Jaspert --- etc/common | 93 +++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 37 deletions(-) (limited to 'etc') diff --git a/etc/common b/etc/common index 78d8451..f0968e8 100644 --- a/etc/common +++ b/etc/common @@ -71,12 +71,19 @@ signal () { date -u >> "${LOGDIR}/${MIRROR}.log" PUSHARGS="" + + # Archive is non-empty, so tell the downstreams if [ -n "${PUSHARCHIVE}" ]; then PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" fi + # We have a callback wish, tell downstreams if [ -n "${PUSHCB}" ]; then PUSHARGS="${PUSHARGS} sync:callback" fi + # If we are running an mhop push AND our downstream is one to receive it, tell it. + if [ "xmhopx" = "x${PUSHKIND}x" ] && [ "xmhopx" = "x${PUSHTYPE}x" ]; then + PUSHARGS="${PUSHARGS} sync:mhop" + fi if [ "xallx" = "x${PUSHTYPE}x" ]; then # Default normal "fire and forget" push @@ -87,48 +94,60 @@ signal () { # Want a staged push. Fine, lets do that echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" - # Step1: Do a push to only sync stage1, do not background - PUSHARGS1="sync:stage1" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 - touch "${PUSHLOCKOWN}" - - # Step2: Wait for all the other "lock"files to appear. - tries=0 - # We do not wait forever - while [ ${tries} -lt ${PUSHDELAY} ]; do - total=0 - found=0 - for file in ${PUSHLOCKS}; do - total=$((total + 1)) - if [ -f ${file} ]; then - found=$((found + 1)) + # Only send stage1 if we havent already send it. When called with stage2, we already did. + if [ "xstage2x" != "x${PUSHKIND}x" ]; then + # Step1: Do a push to only sync stage1, do not background + PUSHARGS1="sync:stage1" + ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + touch "${PUSHLOCKOWN}" + + # Step2: Wait for all the other "lock"files to appear. + tries=0 + # We do not wait forever + while [ ${tries} -lt ${PUSHDELAY} ]; do + total=0 + found=0 + for file in ${PUSHLOCKS}; do + total=$((total + 1)) + if [ -f ${file} ]; then + found=$((found + 1)) + fi + done + if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then + touch "${LOCKDIR}/all_stage1" + break fi + tries=$((tries + 5)) + sleep 5 done - if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then - touch "${LOCKDIR}/all_stage1" - break + # In case we did not have all PUSHLOCKS and still continued, note it + # This is a little racy, especially if the other parts decide to do this + # at the same time, but it wont hurt more than a mail too much, so I don't care much + if [ ${tries} -ge ${PUSHDELAY} ]; then + echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" + for file in ${PUSHLOCKS}; do + if [ ! -f ${file} ]; then + echo "${file}" >> "${LOGDIR}/${MIRROR}.log" + error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" + fi + done fi - tries=$((tries + 5)) - sleep 5 - done - # In case we did not have all PUSHLOCKS and still continued, note it - # This is a little racy, especially if the other parts decide to do this - # at the same time, but it wont hurt more than a mail too much, so I don't care much - if [ ${tries} -ge ${PUSHDELAY} ]; then - echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" - for file in ${PUSHLOCKS}; do - if [ ! -f ${file} ]; then - echo "${file}" >> "${LOGDIR}/${MIRROR}.log" - error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" - fi - done + rm -f "${PUSHLOCKOWN}" fi - rm -f "${PUSHLOCKOWN}" - # Step3: It either timed out or we have all the "lock"files, sync stage2 - PUSHARGS2="sync:stage2" - echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + # Step3: It either timed out or we have all the "lock"files, do the rest + # If we are doing mhop AND are called from ftpsync - we now exit. + # That way we notify our uplink that we and all our clients are done with their + # stage1. It can then finish its own, and if all our upstreams downlinks are done, + # it will send us stage2. + # If we are not doing mhop or are not called from ftpsync, we start stage2 + if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then + return + else + PUSHARGS2="sync:stage2" + echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" + ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + fi else # Can't decide? Then you get nothing. return -- cgit v1.2.3 From ae0c71d67d11c9e70957911dd60c057941fada8b Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Tue, 29 Dec 2009 10:42:42 +0100 Subject: common add a few comments Signed-off-by: Joerg Jaspert --- etc/common | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'etc') diff --git a/etc/common b/etc/common index f0968e8..a45483c 100644 --- a/etc/common +++ b/etc/common @@ -17,6 +17,8 @@ # then push stage2 # $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) # $PUSHCB - do we want a callback? +# $PUSHKIND - whats going on? are we doing mhop push or already stage2? +# $FROMFTPSYNC - set to true if we run from within ftpsync. # # This function assumes that the variable LOG is set to a directory where # logfiles can be written to. @@ -47,23 +49,26 @@ signal () { # Defaults we always want, no matter what SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + # If there are userdefined ssh options, add them. if [ -n "${SSH_OPTS}" ]; then SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}" fi + # Does this machine need a special key? if [ -n "${SSHKEY}" ]; then SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}" fi + # Does this machine have an extra own set of ssh options? if [ -n "${SSHOPTS}" ]; then SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}" fi + # Set the protocol version if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ] && [ ${SSHPROTO} -ne 99 ]; then # Idiots, we only want 1 or 2. Cant decide? Lets force 2. SSHPROTO=2 fi - if [ -n "${SSHPROTO}" ] && [ ${SSHPROTO} -ne 99 ]; then SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}" fi @@ -86,12 +91,12 @@ signal () { fi if [ "xallx" = "x${PUSHTYPE}x" ]; then - # Default normal "fire and forget" push + # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log" PUSHARGS1="sync:all" ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 & elif [ "xstagedx" = "x${PUSHTYPE}x" ]; then - # Want a staged push. Fine, lets do that + # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" # Only send stage1 if we havent already send it. When called with stage2, we already did. -- cgit v1.2.3 From 693d141ebe88d1428eda958ffe6ead7fa164d258 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Tue, 29 Dec 2009 13:18:36 +0100 Subject: More comments in the mirror sample Signed-off-by: Joerg Jaspert --- etc/runmirrors.mirror.sample | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'etc') diff --git a/etc/runmirrors.mirror.sample b/etc/runmirrors.mirror.sample index e2820e3..744ad98 100644 --- a/etc/runmirrors.mirror.sample +++ b/etc/runmirrors.mirror.sample @@ -9,10 +9,13 @@ # # The fields Type, ShortName, HostName and User are *mandantory*. # -# Type is either all or staged, meaning: +# Type is either all, staged or mhop, meaning: # all - do a "normal" push. Trigger them, go on. # staged - do a two-stage push, waiting for them after stage 2(and all others that # are staged) before doing stage2 +# mhop - send a multi-hop staged push. This will tell the mirror to initiate +# a mhop/stage1 push to its staged/mhop mirrors and then exit. +# When all mhop got back we then send the stage2 through to them. # # ShortName will be used as a shorthand in logfile outputs and for the logfile # where every ssh output gets redirected to. @@ -42,18 +45,25 @@ # Examples: # all eu.puccini puccini.debian.org archvsync 2 # -# will push puccini.debian.org, user archvsync, using ssh protocol 2 and the specified ssh key. -# +# -> will push puccini.debian.org, user archvsync, using ssh protocol 2 +# and the globally configured ssh key. # # all eu.puccini puccini.debian.org archvsync -p 2222 # -# which will do the same as above, but use port 2222 to connect to. +# -> will do the same as above, but use port 2222 to connect to. # # staged eu.puccini puccini.debian.org archvsync # staged eu.powell powell.debian.org archvsync # -# will push both puccini and powell in stage1, waiting for both to -# finish stage1 before stage2 gets pushed. The username will be archvsync. +# -> will push both puccini and powell in stage1, waiting for both to +# finish stage1 before stage2 gets pushed. The username will be archvsync. +# +# staged eu.puccini puccini.debian.org archvsync +# mhop eu.powell powell.debian.org archvsync +# +# -> will do the same as above, but powell gets told about mhop and can then +# push its own staged/mhop mirrors before returning. When both returned +# then stage2 is sent to both. # # One special value is allowed: DELAY # This word has to be on a line itself, followed by a space and a number. -- cgit v1.2.3 From 24de4a2f1d314be85bb3eb14c15bd0bf8de015ae Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Wed, 30 Dec 2009 21:25:16 +0100 Subject: common treat mhop mirror entries same as staged in this elif Signed-off-by: Joerg Jaspert --- etc/common | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'etc') diff --git a/etc/common b/etc/common index a45483c..2f6af01 100644 --- a/etc/common +++ b/etc/common @@ -76,7 +76,6 @@ signal () { date -u >> "${LOGDIR}/${MIRROR}.log" PUSHARGS="" - # Archive is non-empty, so tell the downstreams if [ -n "${PUSHARCHIVE}" ]; then PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" @@ -95,7 +94,7 @@ signal () { echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log" PUSHARGS1="sync:all" ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 & - elif [ "xstagedx" = "x${PUSHTYPE}x" ]; then + elif [ "xstagedx" = "x${PUSHTYPE}x" ] || [ "xmhopx" = "x${PUSHTYPE}x" ]; then # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" -- cgit v1.2.3