From 69fd1eda70605488c21f5773a0ba62772e14650d Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 16:52:57 +0100 Subject: common also look at mhop hosts in get2stage Signed-off-by: Joerg Jaspert --- etc/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'etc/common') diff --git a/etc/common b/etc/common index 465ab73..daa228d 100644 --- a/etc/common +++ b/etc/common @@ -181,7 +181,7 @@ hook () { # Return the list of 2-stage mirrors. get2stage() { - egrep '^staged' "${MIRRORS}" | { + egrep '^(staged|mhop)' "${MIRRORS}" | { while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}" done -- cgit v1.2.3 From 32d1612c28af425609306a494f476d31ad8c9d10 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:33:59 +0100 Subject: common 2 new options Signed-off-by: Joerg Jaspert --- etc/common | 2 ++ 1 file changed, 2 insertions(+) (limited to 'etc/common') diff --git a/etc/common b/etc/common index daa228d..78d8451 100644 --- a/etc/common +++ b/etc/common @@ -38,6 +38,8 @@ signal () { PUSHTYPE=${PUSHTYPE:-"all"} PUSHARCHIVE=${PUSHARCHIVE:-""} PUSHCB=${PUSHCB:-""} + PUSHKIND=${PUSHKIND:-"all"} + FROMFTPSYNC=${FROMFTPSYNC:-"false"} # And now get # back to space... SSHOPTS=${SSHOPTS/\#/ } -- cgit v1.2.3 From 9c0dad2d25d7abbd9842d009483e528d9a0515e4 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:34:24 +0100 Subject: common properly signal the mhop foo Signed-off-by: Joerg Jaspert --- etc/common | 93 +++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 37 deletions(-) (limited to 'etc/common') diff --git a/etc/common b/etc/common index 78d8451..f0968e8 100644 --- a/etc/common +++ b/etc/common @@ -71,12 +71,19 @@ signal () { date -u >> "${LOGDIR}/${MIRROR}.log" PUSHARGS="" + + # Archive is non-empty, so tell the downstreams if [ -n "${PUSHARCHIVE}" ]; then PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" fi + # We have a callback wish, tell downstreams if [ -n "${PUSHCB}" ]; then PUSHARGS="${PUSHARGS} sync:callback" fi + # If we are running an mhop push AND our downstream is one to receive it, tell it. + if [ "xmhopx" = "x${PUSHKIND}x" ] && [ "xmhopx" = "x${PUSHTYPE}x" ]; then + PUSHARGS="${PUSHARGS} sync:mhop" + fi if [ "xallx" = "x${PUSHTYPE}x" ]; then # Default normal "fire and forget" push @@ -87,48 +94,60 @@ signal () { # Want a staged push. Fine, lets do that echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" - # Step1: Do a push to only sync stage1, do not background - PUSHARGS1="sync:stage1" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 - touch "${PUSHLOCKOWN}" - - # Step2: Wait for all the other "lock"files to appear. - tries=0 - # We do not wait forever - while [ ${tries} -lt ${PUSHDELAY} ]; do - total=0 - found=0 - for file in ${PUSHLOCKS}; do - total=$((total + 1)) - if [ -f ${file} ]; then - found=$((found + 1)) + # Only send stage1 if we havent already send it. When called with stage2, we already did. + if [ "xstage2x" != "x${PUSHKIND}x" ]; then + # Step1: Do a push to only sync stage1, do not background + PUSHARGS1="sync:stage1" + ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + touch "${PUSHLOCKOWN}" + + # Step2: Wait for all the other "lock"files to appear. + tries=0 + # We do not wait forever + while [ ${tries} -lt ${PUSHDELAY} ]; do + total=0 + found=0 + for file in ${PUSHLOCKS}; do + total=$((total + 1)) + if [ -f ${file} ]; then + found=$((found + 1)) + fi + done + if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then + touch "${LOCKDIR}/all_stage1" + break fi + tries=$((tries + 5)) + sleep 5 done - if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then - touch "${LOCKDIR}/all_stage1" - break + # In case we did not have all PUSHLOCKS and still continued, note it + # This is a little racy, especially if the other parts decide to do this + # at the same time, but it wont hurt more than a mail too much, so I don't care much + if [ ${tries} -ge ${PUSHDELAY} ]; then + echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" + for file in ${PUSHLOCKS}; do + if [ ! -f ${file} ]; then + echo "${file}" >> "${LOGDIR}/${MIRROR}.log" + error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" + fi + done fi - tries=$((tries + 5)) - sleep 5 - done - # In case we did not have all PUSHLOCKS and still continued, note it - # This is a little racy, especially if the other parts decide to do this - # at the same time, but it wont hurt more than a mail too much, so I don't care much - if [ ${tries} -ge ${PUSHDELAY} ]; then - echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" - for file in ${PUSHLOCKS}; do - if [ ! -f ${file} ]; then - echo "${file}" >> "${LOGDIR}/${MIRROR}.log" - error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" - fi - done + rm -f "${PUSHLOCKOWN}" fi - rm -f "${PUSHLOCKOWN}" - # Step3: It either timed out or we have all the "lock"files, sync stage2 - PUSHARGS2="sync:stage2" - echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + # Step3: It either timed out or we have all the "lock"files, do the rest + # If we are doing mhop AND are called from ftpsync - we now exit. + # That way we notify our uplink that we and all our clients are done with their + # stage1. It can then finish its own, and if all our upstreams downlinks are done, + # it will send us stage2. + # If we are not doing mhop or are not called from ftpsync, we start stage2 + if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then + return + else + PUSHARGS2="sync:stage2" + echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" + ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + fi else # Can't decide? Then you get nothing. return -- cgit v1.2.3 From ae0c71d67d11c9e70957911dd60c057941fada8b Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Tue, 29 Dec 2009 10:42:42 +0100 Subject: common add a few comments Signed-off-by: Joerg Jaspert --- etc/common | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'etc/common') diff --git a/etc/common b/etc/common index f0968e8..a45483c 100644 --- a/etc/common +++ b/etc/common @@ -17,6 +17,8 @@ # then push stage2 # $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) # $PUSHCB - do we want a callback? +# $PUSHKIND - whats going on? are we doing mhop push or already stage2? +# $FROMFTPSYNC - set to true if we run from within ftpsync. # # This function assumes that the variable LOG is set to a directory where # logfiles can be written to. @@ -47,23 +49,26 @@ signal () { # Defaults we always want, no matter what SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + # If there are userdefined ssh options, add them. if [ -n "${SSH_OPTS}" ]; then SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}" fi + # Does this machine need a special key? if [ -n "${SSHKEY}" ]; then SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}" fi + # Does this machine have an extra own set of ssh options? if [ -n "${SSHOPTS}" ]; then SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}" fi + # Set the protocol version if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ] && [ ${SSHPROTO} -ne 99 ]; then # Idiots, we only want 1 or 2. Cant decide? Lets force 2. SSHPROTO=2 fi - if [ -n "${SSHPROTO}" ] && [ ${SSHPROTO} -ne 99 ]; then SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}" fi @@ -86,12 +91,12 @@ signal () { fi if [ "xallx" = "x${PUSHTYPE}x" ]; then - # Default normal "fire and forget" push + # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log" PUSHARGS1="sync:all" ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 & elif [ "xstagedx" = "x${PUSHTYPE}x" ]; then - # Want a staged push. Fine, lets do that + # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" # Only send stage1 if we havent already send it. When called with stage2, we already did. -- cgit v1.2.3 From 24de4a2f1d314be85bb3eb14c15bd0bf8de015ae Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Wed, 30 Dec 2009 21:25:16 +0100 Subject: common treat mhop mirror entries same as staged in this elif Signed-off-by: Joerg Jaspert --- etc/common | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'etc/common') diff --git a/etc/common b/etc/common index a45483c..2f6af01 100644 --- a/etc/common +++ b/etc/common @@ -76,7 +76,6 @@ signal () { date -u >> "${LOGDIR}/${MIRROR}.log" PUSHARGS="" - # Archive is non-empty, so tell the downstreams if [ -n "${PUSHARCHIVE}" ]; then PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" @@ -95,7 +94,7 @@ signal () { echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log" PUSHARGS1="sync:all" ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 & - elif [ "xstagedx" = "x${PUSHTYPE}x" ]; then + elif [ "xstagedx" = "x${PUSHTYPE}x" ] || [ "xmhopx" = "x${PUSHTYPE}x" ]; then # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" -- cgit v1.2.3