From c8b0797f49ed9af655f5c7a6e57ffd2075fdb293 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 16:22:48 +0100 Subject: ftpsync add new option sync:mhop Signed-off-by: Joerg Jaspert --- bin/ftpsync | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/ftpsync b/bin/ftpsync index c510f36..b6d9d95 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -56,6 +56,7 @@ VERSION="80286" # stage1 Only do stage1 sync # stage2 Only do stage2 sync # all Do a complete sync +# mhop Do a mhop sync, usually additionally to stage1 # archive:foo Sync archive foo (if config for foo is available) # callback Call back when done (needs proper ssh setup for this to # work). It will always use the "command" callback:$HOSTNAME @@ -89,6 +90,9 @@ check_commandline() { sync:all) SYNCALL="true" ;; + sync:mhop) + SYNCMHOP="true" + ;; *) echo "Unknown option ${1} ignored" ;; @@ -280,6 +284,8 @@ SYNCSTAGE1=${SYNCSTAGE1:-"false"} SYNCSTAGE2=${SYNCSTAGE2:-"false"} # Do we sync all? SYNCALL=${SYNCALL:-"true"} +# Do we have a mhop sync? +SYNCMHOP=${SYNCMHOP:-"false"} # Do we callback? SYNCCALLBACK=${SYNCCALLBACK:-"false"} # If we call back we need some more options defined in the config file. -- cgit v1.2.3 From 5533b44c10b35725d13276b1d7f3194250ac4add Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 16:47:20 +0100 Subject: ftpsync. call runmirrors with getopts args call runmirrors with the getopts args it has now. always present -a and -f. only set -s and -k when appropriate. Signed-off-by: Joerg Jaspert --- bin/ftpsync | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index b6d9d95..2808d34 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -458,18 +458,33 @@ function mainroutine() { rm -f "${LOCK}" if [ x${HUB} = "xtrue" ]; then - # Only trigger slave mirrors if we had a push for stage2 or all - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - log "Trigger slave mirrors" - ${BASEDIR}/bin/runmirrors "${ARCHIVE}" - log "Trigger slave done" - - HOOK=( - HOOKNR=5 - HOOKSCR=${HOOK5} - ) - hook $HOOK - fi + # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then + # We tell runmirrors about the archive we are running on. Might be empty. + RUNMIRRORARGS="-a ${ARCHIVE}" + # We also tell runmirrors that we are running it from within ftpsync, so it can change + # the way it works with mhop based on that. + RUNMIRRORARGS="${RUNMIRRORARGS} -f" + + if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then + # This is true when we have a mhop sync. And only then, due to the earlier if. + # So if that happens, tell runmirrors we are doing mhop stage1. + RUNMIRRORARGS="${RUNMIRRORARGS} -s stage1 -k mhop" + elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -s stage2" + elif [ "xtruex" = "x${SYNCALL}x" ]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -s all" + fi + log "Trigger slave mirrors" + ${BASEDIR}/bin/runmirrors "${RUNMIRRORARGS}" + log "Trigger slave done" + + HOOK=( + HOOKNR=5 + HOOKSCR=${HOOK5} + ) + hook $HOOK + fi fi # All done, lets call cleanup -- cgit v1.2.3 From f59c7cea77737a2ff5e8c14ab42dc8d6a55d693f Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 16:51:33 +0100 Subject: runmirrors, getopts add new option parsing using getopts. keep the old way of just telling it the archive name working Signed-off-by: Joerg Jaspert --- bin/runmirrors | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/bin/runmirrors b/bin/runmirrors index a79e33d..3a9d943 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -28,11 +28,69 @@ BASEDIR=${BASEDIR:-"${HOME}"} NAME="`basename $0`" -# In case we are called with an argument we look for a different configuration. -CONF=${1:-""} -if [ -n "${CONF}" ]; then - NAME="${NAME}-${CONF}" +HELP="$0, (C) 2008, 2009 by Joerg Jaspert \n +Usage:\n\n + +1.) a single parameter with NO leading -.\n +\t This will will then be used as the addition for our configfile. Ie. \`$0 security\` will\n +\t have us look for ${NAME}-security.{conf,mirror} files.\n\n + +2.) using getopt style parameters:\n +\t -a [NAME] - Same as 1.) above, used for the config files. Default empty.\n +\t -k [TYPE] - Type of push. all, mhop. Default all.\n +\t -s [STAGE] - Stage of push. stage1, stage2, all. Default all.\n +\t -f - Run from within the mirrorscript ftpsync. Don't use from commandline!\n +\t -h - Print this help and exit +" +# If we got options, lets see if we use newstyle options, or oldstyle. If oldstyle +# it will not start with a -. If we find oldstyle we assume its only one, the config +# name we run on. +if [ $# -gt 0 ]; then + if [ "x${1:0:1}x" != "x-x" ]; then + # Yes, does not start with a -, so use it for the config name. + CONF=${1:-""} + if [ -n "${CONF}" ]; then + NAME="${NAME}-${CONF}" + fi + else + # Yeah well, new style, starting with - for getopts + while getopts ':a:k:fh' OPTION ; do + case $OPTION in + a) CONF=${OPTARG} + NAME=${NAME}-${CONF} + ;; + k) PUSHKIND=${OPTARG} + ;; + f) FROMFTPSYNC="true" + ;; + h) echo -e $HELP + exit 0 + ;; + + \?) echo "Unknown option \"-$OPTARG\"." + echo -e $HELP + exit 1 + ;; + :) echo "Option \"-$OPTARG\" needs an argument." + echo -e $HELP + exit 1 + ;; + esac + done + fi fi +# Make sure the values are always defined, even if there was no commandline option +# for them +# Default config is empty +CONF=${CONF:-""} + +# Set the default to all, if we didnt get told about it. Currently +# valid: all - normal push. mhop - multi-hop multi-stage push. Default is all. +# If mhop we will run a normal stage1 push *and then exit*. +PUSHKIND=${PUSHKIND:-"all"} + +# If we are pushed from within ftpsync. Default false. +FROMFTPSYNC=${FROMFTPSYNC:-"false"} # Read our config file . "${BASEDIR}/etc/${NAME}.conf" -- cgit v1.2.3 From 2d67be7b5cefd83acd188f640d7b3a582f8b45ea Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 16:51:57 +0100 Subject: runmirrors add two seperators between initial setup, config values and code. Signed-off-by: Joerg Jaspert --- bin/runmirrors | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/runmirrors b/bin/runmirrors index 3a9d943..413738f 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -92,6 +92,7 @@ PUSHKIND=${PUSHKIND:-"all"} # If we are pushed from within ftpsync. Default false. FROMFTPSYNC=${FROMFTPSYNC:-"false"} +######################################################################## # Read our config file . "${BASEDIR}/etc/${NAME}.conf" @@ -138,6 +139,8 @@ HOOK1=${HOOK1:-""} HOOK2=${HOOK2:-""} HOOK3=${HOOK3:-""} +######################################################################## + # Some sane defaults cd ${BASEDIR} umask 022 -- cgit v1.2.3 From 69fd1eda70605488c21f5773a0ba62772e14650d Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 16:52:57 +0100 Subject: common also look at mhop hosts in get2stage Signed-off-by: Joerg Jaspert --- etc/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/common b/etc/common index 465ab73..daa228d 100644 --- a/etc/common +++ b/etc/common @@ -181,7 +181,7 @@ hook () { # Return the list of 2-stage mirrors. get2stage() { - egrep '^staged' "${MIRRORS}" | { + egrep '^(staged|mhop)' "${MIRRORS}" | { while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}" done -- cgit v1.2.3 From 9aaa0aadba78e8ca6fb3fa47839a066b938ac327 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 16:54:53 +0100 Subject: Sample config make clear we need the username in the mirrors file, fix the broken example. Also mention backwards compatibility Signed-off-by: Joerg Jaspert --- etc/runmirrors.mirror.sample | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/etc/runmirrors.mirror.sample b/etc/runmirrors.mirror.sample index 2f841bb..e2820e3 100644 --- a/etc/runmirrors.mirror.sample +++ b/etc/runmirrors.mirror.sample @@ -7,6 +7,7 @@ # # Type ShortName HostName User -$SOMESSHOPTION # +# The fields Type, ShortName, HostName and User are *mandantory*. # # Type is either all or staged, meaning: # all - do a "normal" push. Trigger them, go on. @@ -33,6 +34,11 @@ # want. There is currently no way around this, as that would mean # dropping backward compatibility. # +# Backwards compatibility: +# An older runmirrors script will NOT run with a newer runmirrors.mirror file, but +# a new runmirrors can run with an old runmirrors.mirror file. This should make updates +# painless. +# # Examples: # all eu.puccini puccini.debian.org archvsync 2 # @@ -43,11 +49,11 @@ # # which will do the same as above, but use port 2222 to connect to. # -# staged eu.puccini puccini.debian.org -# staged eu.powell powell.debian.org +# staged eu.puccini puccini.debian.org archvsync +# staged eu.powell powell.debian.org archvsync # # will push both puccini and powell in stage1, waiting for both to -# finish stage1 before stage2 gets pushed. +# finish stage1 before stage2 gets pushed. The username will be archvsync. # # One special value is allowed: DELAY # This word has to be on a line itself, followed by a space and a number. -- cgit v1.2.3 From 8705fdd787775366c3d3e5795f9428e24720e5a4 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:31:45 +0100 Subject: ftpsync call runmirrors with the correct argument set Signed-off-by: Joerg Jaspert --- bin/ftpsync | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index 2808d34..5e127cd 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -467,13 +467,14 @@ function mainroutine() { RUNMIRRORARGS="${RUNMIRRORARGS} -f" if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then - # This is true when we have a mhop sync. And only then, due to the earlier if. - # So if that happens, tell runmirrors we are doing mhop stage1. - RUNMIRRORARGS="${RUNMIRRORARGS} -s stage1 -k mhop" + # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will + # not get to this point. + # So if that happens, tell runmirrors we are doing mhop + RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -s stage2" + RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" elif [ "xtruex" = "x${SYNCALL}x" ]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -s all" + RUNMIRRORARGS="${RUNMIRRORARGS} -k all" fi log "Trigger slave mirrors" ${BASEDIR}/bin/runmirrors "${RUNMIRRORARGS}" -- cgit v1.2.3 From 2863757b0fb80c151daf8d6e4788ba14495f8b4c Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:32:01 +0100 Subject: runmirrors drop the -s Signed-off-by: Joerg Jaspert --- bin/runmirrors | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/runmirrors b/bin/runmirrors index 413738f..791ee16 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -37,8 +37,7 @@ Usage:\n\n 2.) using getopt style parameters:\n \t -a [NAME] - Same as 1.) above, used for the config files. Default empty.\n -\t -k [TYPE] - Type of push. all, mhop. Default all.\n -\t -s [STAGE] - Stage of push. stage1, stage2, all. Default all.\n +\t -k [TYPE] - Type of push. all, stage2, mhop. Default all.\n \t -f - Run from within the mirrorscript ftpsync. Don't use from commandline!\n \t -h - Print this help and exit " -- cgit v1.2.3 From 05c7958f9f9dd12d5f802d679d69c651699f23b3 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:32:48 +0100 Subject: runmirrors only look at mhop/&taged entries if doing mhop push Signed-off-by: Joerg Jaspert --- bin/runmirrors | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bin/runmirrors b/bin/runmirrors index 791ee16..23c2e32 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -84,8 +84,8 @@ fi CONF=${CONF:-""} # Set the default to all, if we didnt get told about it. Currently -# valid: all - normal push. mhop - multi-hop multi-stage push. Default is all. -# If mhop we will run a normal stage1 push *and then exit*. +# valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1, +# stage2 - staged push, second phase. Default is all. PUSHKIND=${PUSHKIND:-"all"} # If we are pushed from within ftpsync. Default false. @@ -183,6 +183,14 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do continue fi + # If we are told we have a mhop sync to do, we will only look at staged/mhop + # entries and ignore the rest. + if [ "x${PUSHKIND}x" = "xmhopx" ]; then + if [ "x${MTYPE}x" != "xstagedx" ] || [ "x${MTYPE}x" != "xmhopx" ]; then + continue + fi + fi + # Now, MSSHOPT may start with a -. In that case the whole rest of the line is taken # as a set of options to give to ssh, we pass it without doing anything with it. # If it starts with a 1 or 2 then it will tell us about the ssh protocol version to use, -- cgit v1.2.3 From 49607174263e88a82d16d131bf4ea3cac71d5c91 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:33:42 +0100 Subject: runmirrors two new options in the array Signed-off-by: Joerg Jaspert --- bin/runmirrors | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/bin/runmirrors b/bin/runmirrors index 23c2e32..60471d6 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -231,15 +231,17 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do # Built our array SIGNAL_OPTS=( - MIRROR="${MLNAME}" - HOSTNAME="${MHOSTNAME}" - USERNAME="${MUSER}" - SSHPROTO="${MPROTO}" - SSHKEY="${MKEYFILE}" - SSHOPTS="${SSHOPT/ /#}" - PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1" - PUSHTYPE="${MTYPE}" - PUSHARCHIVE=${PUSHARCHIVE} + MIRROR="${MLNAME}" + HOSTNAME="${MHOSTNAME}" + USERNAME="${MUSER}" + SSHPROTO="${MPROTO}" + SSHKEY="${MKEYFILE}" + SSHOPTS="${SSHOPT/ /#}" + PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1" + PUSHTYPE="${MTYPE}" + PUSHARCHIVE=${PUSHARCHIVE} + PUSHKIND=${PUSHKIND} + FROMFTPSYNC=${FROMFTPSYNC} ) # And finally, push the mirror -- cgit v1.2.3 From 32d1612c28af425609306a494f476d31ad8c9d10 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:33:59 +0100 Subject: common 2 new options Signed-off-by: Joerg Jaspert --- etc/common | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/common b/etc/common index daa228d..78d8451 100644 --- a/etc/common +++ b/etc/common @@ -38,6 +38,8 @@ signal () { PUSHTYPE=${PUSHTYPE:-"all"} PUSHARCHIVE=${PUSHARCHIVE:-""} PUSHCB=${PUSHCB:-""} + PUSHKIND=${PUSHKIND:-"all"} + FROMFTPSYNC=${FROMFTPSYNC:-"false"} # And now get # back to space... SSHOPTS=${SSHOPTS/\#/ } -- cgit v1.2.3 From 9c0dad2d25d7abbd9842d009483e528d9a0515e4 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:34:24 +0100 Subject: common properly signal the mhop foo Signed-off-by: Joerg Jaspert --- etc/common | 93 +++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/etc/common b/etc/common index 78d8451..f0968e8 100644 --- a/etc/common +++ b/etc/common @@ -71,12 +71,19 @@ signal () { date -u >> "${LOGDIR}/${MIRROR}.log" PUSHARGS="" + + # Archive is non-empty, so tell the downstreams if [ -n "${PUSHARCHIVE}" ]; then PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" fi + # We have a callback wish, tell downstreams if [ -n "${PUSHCB}" ]; then PUSHARGS="${PUSHARGS} sync:callback" fi + # If we are running an mhop push AND our downstream is one to receive it, tell it. + if [ "xmhopx" = "x${PUSHKIND}x" ] && [ "xmhopx" = "x${PUSHTYPE}x" ]; then + PUSHARGS="${PUSHARGS} sync:mhop" + fi if [ "xallx" = "x${PUSHTYPE}x" ]; then # Default normal "fire and forget" push @@ -87,48 +94,60 @@ signal () { # Want a staged push. Fine, lets do that echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" - # Step1: Do a push to only sync stage1, do not background - PUSHARGS1="sync:stage1" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 - touch "${PUSHLOCKOWN}" - - # Step2: Wait for all the other "lock"files to appear. - tries=0 - # We do not wait forever - while [ ${tries} -lt ${PUSHDELAY} ]; do - total=0 - found=0 - for file in ${PUSHLOCKS}; do - total=$((total + 1)) - if [ -f ${file} ]; then - found=$((found + 1)) + # Only send stage1 if we havent already send it. When called with stage2, we already did. + if [ "xstage2x" != "x${PUSHKIND}x" ]; then + # Step1: Do a push to only sync stage1, do not background + PUSHARGS1="sync:stage1" + ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + touch "${PUSHLOCKOWN}" + + # Step2: Wait for all the other "lock"files to appear. + tries=0 + # We do not wait forever + while [ ${tries} -lt ${PUSHDELAY} ]; do + total=0 + found=0 + for file in ${PUSHLOCKS}; do + total=$((total + 1)) + if [ -f ${file} ]; then + found=$((found + 1)) + fi + done + if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then + touch "${LOCKDIR}/all_stage1" + break fi + tries=$((tries + 5)) + sleep 5 done - if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then - touch "${LOCKDIR}/all_stage1" - break + # In case we did not have all PUSHLOCKS and still continued, note it + # This is a little racy, especially if the other parts decide to do this + # at the same time, but it wont hurt more than a mail too much, so I don't care much + if [ ${tries} -ge ${PUSHDELAY} ]; then + echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" + for file in ${PUSHLOCKS}; do + if [ ! -f ${file} ]; then + echo "${file}" >> "${LOGDIR}/${MIRROR}.log" + error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" + fi + done fi - tries=$((tries + 5)) - sleep 5 - done - # In case we did not have all PUSHLOCKS and still continued, note it - # This is a little racy, especially if the other parts decide to do this - # at the same time, but it wont hurt more than a mail too much, so I don't care much - if [ ${tries} -ge ${PUSHDELAY} ]; then - echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" - for file in ${PUSHLOCKS}; do - if [ ! -f ${file} ]; then - echo "${file}" >> "${LOGDIR}/${MIRROR}.log" - error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing" - fi - done + rm -f "${PUSHLOCKOWN}" fi - rm -f "${PUSHLOCKOWN}" - # Step3: It either timed out or we have all the "lock"files, sync stage2 - PUSHARGS2="sync:stage2" - echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" - ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + # Step3: It either timed out or we have all the "lock"files, do the rest + # If we are doing mhop AND are called from ftpsync - we now exit. + # That way we notify our uplink that we and all our clients are done with their + # stage1. It can then finish its own, and if all our upstreams downlinks are done, + # it will send us stage2. + # If we are not doing mhop or are not called from ftpsync, we start stage2 + if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then + return + else + PUSHARGS2="sync:stage2" + echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log" + ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 + fi else # Can't decide? Then you get nothing. return -- cgit v1.2.3 From 422a8bf4de1a4662268413ee32b8e21e17b77c50 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Mon, 28 Dec 2009 22:51:37 +0100 Subject: runmirrors getopts merge \?) and :) into *) Signed-off-by: Joerg Jaspert --- bin/runmirrors | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/runmirrors b/bin/runmirrors index 60471d6..4a3ef51 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -66,12 +66,8 @@ if [ $# -gt 0 ]; then exit 0 ;; - \?) echo "Unknown option \"-$OPTARG\"." - echo -e $HELP - exit 1 - ;; - :) echo "Option \"-$OPTARG\" needs an argument." - echo -e $HELP + *) echo "Invalid usage" + echo -e $HELP exit 1 ;; esac -- cgit v1.2.3 From a6550370967800a9c365d0ad9b65b3e9553a7e7b Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Tue, 29 Dec 2009 10:36:10 +0100 Subject: set delay to 600 to be what we tell users in our config file would be default Signed-off-by: Joerg Jaspert --- bin/runmirrors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/runmirrors b/bin/runmirrors index 4a3ef51..6952228 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -172,7 +172,7 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do if [ "x${MTYPE}x" = "xDELAYx" ]; then # We should wait a bit. if [ -z ${MLNAME} ]; then - MLNAME=60 + MLNAME=600 fi log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}" sleep ${MLNAME} -- cgit v1.2.3 From ae0c71d67d11c9e70957911dd60c057941fada8b Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Tue, 29 Dec 2009 10:42:42 +0100 Subject: common add a few comments Signed-off-by: Joerg Jaspert --- etc/common | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/etc/common b/etc/common index f0968e8..a45483c 100644 --- a/etc/common +++ b/etc/common @@ -17,6 +17,8 @@ # then push stage2 # $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) # $PUSHCB - do we want a callback? +# $PUSHKIND - whats going on? are we doing mhop push or already stage2? +# $FROMFTPSYNC - set to true if we run from within ftpsync. # # This function assumes that the variable LOG is set to a directory where # logfiles can be written to. @@ -47,23 +49,26 @@ signal () { # Defaults we always want, no matter what SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + # If there are userdefined ssh options, add them. if [ -n "${SSH_OPTS}" ]; then SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}" fi + # Does this machine need a special key? if [ -n "${SSHKEY}" ]; then SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}" fi + # Does this machine have an extra own set of ssh options? if [ -n "${SSHOPTS}" ]; then SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}" fi + # Set the protocol version if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ] && [ ${SSHPROTO} -ne 99 ]; then # Idiots, we only want 1 or 2. Cant decide? Lets force 2. SSHPROTO=2 fi - if [ -n "${SSHPROTO}" ] && [ ${SSHPROTO} -ne 99 ]; then SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}" fi @@ -86,12 +91,12 @@ signal () { fi if [ "xallx" = "x${PUSHTYPE}x" ]; then - # Default normal "fire and forget" push + # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log" PUSHARGS1="sync:all" ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 & elif [ "xstagedx" = "x${PUSHTYPE}x" ]; then - # Want a staged push. Fine, lets do that + # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" # Only send stage1 if we havent already send it. When called with stage2, we already did. -- cgit v1.2.3 From 693d141ebe88d1428eda958ffe6ead7fa164d258 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Tue, 29 Dec 2009 13:18:36 +0100 Subject: More comments in the mirror sample Signed-off-by: Joerg Jaspert --- etc/runmirrors.mirror.sample | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/etc/runmirrors.mirror.sample b/etc/runmirrors.mirror.sample index e2820e3..744ad98 100644 --- a/etc/runmirrors.mirror.sample +++ b/etc/runmirrors.mirror.sample @@ -9,10 +9,13 @@ # # The fields Type, ShortName, HostName and User are *mandantory*. # -# Type is either all or staged, meaning: +# Type is either all, staged or mhop, meaning: # all - do a "normal" push. Trigger them, go on. # staged - do a two-stage push, waiting for them after stage 2(and all others that # are staged) before doing stage2 +# mhop - send a multi-hop staged push. This will tell the mirror to initiate +# a mhop/stage1 push to its staged/mhop mirrors and then exit. +# When all mhop got back we then send the stage2 through to them. # # ShortName will be used as a shorthand in logfile outputs and for the logfile # where every ssh output gets redirected to. @@ -42,18 +45,25 @@ # Examples: # all eu.puccini puccini.debian.org archvsync 2 # -# will push puccini.debian.org, user archvsync, using ssh protocol 2 and the specified ssh key. -# +# -> will push puccini.debian.org, user archvsync, using ssh protocol 2 +# and the globally configured ssh key. # # all eu.puccini puccini.debian.org archvsync -p 2222 # -# which will do the same as above, but use port 2222 to connect to. +# -> will do the same as above, but use port 2222 to connect to. # # staged eu.puccini puccini.debian.org archvsync # staged eu.powell powell.debian.org archvsync # -# will push both puccini and powell in stage1, waiting for both to -# finish stage1 before stage2 gets pushed. The username will be archvsync. +# -> will push both puccini and powell in stage1, waiting for both to +# finish stage1 before stage2 gets pushed. The username will be archvsync. +# +# staged eu.puccini puccini.debian.org archvsync +# mhop eu.powell powell.debian.org archvsync +# +# -> will do the same as above, but powell gets told about mhop and can then +# push its own staged/mhop mirrors before returning. When both returned +# then stage2 is sent to both. # # One special value is allowed: DELAY # This word has to be on a line itself, followed by a space and a number. -- cgit v1.2.3 From eacefd19d327e2257fdd882a1b7795dad580f39d Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Tue, 29 Dec 2009 13:36:18 +0100 Subject: readme update readme and mention multi-hop Signed-off-by: Joerg Jaspert --- README | 61 +++++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/README b/README index 8541eaf..e85af02 100644 --- a/README +++ b/README @@ -11,7 +11,7 @@ Currently the following scripts are available: * ftpsync - Used to sync an archive using rsync * runmirrors - Used to notify leaf nodes of available updates - * dircombine - Internal script to manage the mirror user's $HOME + * dircombine - Internal script to manage the mirror user's $HOME on debian.org machines * typicalsync - Generates a typical Debian mirror * udh - We are lazy, just a shorthand to avoid typing the @@ -23,14 +23,14 @@ For impatient people, short usage instruction: - Create a dedicated user for the whole mirror. - Create a seperate directory for the mirror, writeable by the new user. - - Place the ftpsync script in the mirror user's $HOME/ (or $HOME/bin) + - Place the ftpsync script in the mirror user's $HOME/bin (or just $HOME) - Place the ftpsync.conf.sample into $HOME/etc as ftpsync.conf and edit - it to suit your system. You should at the very least change the TO= + it to suit your system. You should at the very least change the TO= and RSYNC_HOST lines. - Create $HOME/log (or wherever you point $LOGDIR to) - Setup the .ssh/authorized_keys for the mirror user and place the public key of your upstream mirror into it. Preface it with -no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="~/bin/ftpsync",from="IPADDRESS" +no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="~/bin/ftpsync",from="IPADDRESS" and replace $IPADDRESS with that of your upstream mirror. - You are finished @@ -53,7 +53,7 @@ functionality: - Must perform a 2-stage sync The archive mirroring must be done in 2 stages. The first rsync run - must ignore the index files. The correct exclude options for the + must ignore the index files. The correct exclude options for the first rsync run are: --exclude Packages* --exclude Sources* --exclude Release* --exclude ls-lR* The first stage must not delete any files. @@ -76,7 +76,7 @@ functionality: - Should understand multi-stage pushes. The script should parse the arguments it gets via ssh, and if they - contain a hint to only sync stage1 or stage2, then ONLY those steps + contain a hint to only sync stage1 or stage2, then ONLY those steps SHOULD be performed. Rationale: This enables us to coordinate the timing of the first @@ -106,51 +106,67 @@ Some of the advantages of the new version are: - Support for multi-stage archive synchronisations - Support for hook scripts at various points - Support for multiple archives, even if they are pushed using one ssh key + - Support for multi-hop, multi-stage archive synchronisations Correct support for multiple pushes ----------------------------------- When the script receives a second push while it is running and syncing - the archive it won't ignore it. Instead it will rerun the + the archive it won't ignore it. Instead it will rerun the synchronisation step to ensure the archive is correctly synchronised. - + Scripts that fail to do that risk ending up with an inconsistent archive. - - + + Can do multi-stage archive synchronisations ------------------------------------------- The script can be told to only perform the first or second stage of the - archive synchronisation. - + archive synchronisation. + This enables us to send all the binary packages and sources to a number of mirrors, and then tell all of them to sync the Packages/Release files at once. This will keep the timeframe in which the mirrors are out of sync very small and will greatly help things like DNS RR entries or even the planned GeoDNS setup. - - + + + Multi-hop, multi-stage archive synchronisations + ----------------------------------------------- + The script can be told to perform a multi-hop multi-stage archive + synchronisation. + + This is basically the same as the multi-stage synchronisation + explained above, but enables the downstream mirror to push his own + staged/multi-hop downstreams before returning. This has the same + advantage than the multi-stage synchronisation but allows us to do + this over multiple level of mirrors. (Imagine one push going from + Europe to Australia, where then locally 3 others get updated before + stage2 is sent out. Instead of 4times transferring data from Europe to + Australia, just to have them all updated near instantly). + + Can run hook scripts -------------------- ftpsync currently allows 5 hook scripts to run at various points of the mirror sync run. - + Hook1: After lock is acquired, before first rsync Hook2: After first rsync, if successful Hook3: After second rsync, if successful Hook4: Right before leaf mirror triggering Hook5: After leaf mirror trigger (only if we have slave mirrors; HUB=true) - + Note that Hook3 and Hook4 are likely to be called directly after each other. The difference is that Hook3 is called *every* time the second rsync succeeds even if the mirroring needs to re-run due to a second push. Hook4 is only executed if mirroring is completed. - - + + Support for multiple archives, even if they are pushed using one ssh key ------------------------------------------------------------------------ If you get multiple archives from your upstream mirror (say Debian, - Debian-Backports and Volatile), previously you had to use 3 different ssh + Debian-Backports and Volatile), previously you had to use 3 different ssh keys to be able to automagically synchronize them. This script can do it - all with just one key, if your upstream mirror tells you which archive. + all with just one key, if your upstream mirror tells you which archive. See "Commandline/SSH options" below for further details. @@ -171,6 +187,7 @@ Option Behaviour stage1 Only do stage1 sync stage2 Only do stage2 sync all Do a complete sync (default) +mhop Do a multi-hop sync archive:foo Sync archive foo (if the file $HOME/etc/ftpsync-foo.conf exists and is configured) callback Call back when done (needs proper ssh setup for this to @@ -223,9 +240,9 @@ as well as the newer multi-stage push. The normal push, as described above, will simply push the leaf node and then go on with the other nodes. -The multi-staged push first pushes a mirror and tells it to only do a +The multi-staged push first pushes a mirror and tells it to only do a stage1 sync run. Then it waits for the mirror (and all others being pushed -in the same run) to finish that run, before it tells all of the staged +in the same run) to finish that run, before it tells all of the staged mirrors to do the stage2 sync. This way you can do a nearly-simultaneous update of multiple hosts. -- cgit v1.2.3 From 537ed442d099e95a3831a7d2ce1496f725f310b1 Mon Sep 17 00:00:00 2001 From: Uli Martens Date: Tue, 29 Dec 2009 18:11:20 +0100 Subject: fix whitespace to minimize diff between master and mhop branches Signed-off-by: Joerg Jaspert --- bin/ftpsync | 56 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index 5e127cd..0d5a2ed 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -458,34 +458,34 @@ function mainroutine() { rm -f "${LOCK}" if [ x${HUB} = "xtrue" ]; then - # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop - if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then - # We tell runmirrors about the archive we are running on. Might be empty. - RUNMIRRORARGS="-a ${ARCHIVE}" - # We also tell runmirrors that we are running it from within ftpsync, so it can change - # the way it works with mhop based on that. - RUNMIRRORARGS="${RUNMIRRORARGS} -f" - - if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then - # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will - # not get to this point. - # So if that happens, tell runmirrors we are doing mhop - RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" - elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" - elif [ "xtruex" = "x${SYNCALL}x" ]; then - RUNMIRRORARGS="${RUNMIRRORARGS} -k all" - fi - log "Trigger slave mirrors" - ${BASEDIR}/bin/runmirrors "${RUNMIRRORARGS}" - log "Trigger slave done" - - HOOK=( - HOOKNR=5 - HOOKSCR=${HOOK5} - ) - hook $HOOK - fi + # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then + # We tell runmirrors about the archive we are running on. Might be empty. + RUNMIRRORARGS="-a ${ARCHIVE}" + # We also tell runmirrors that we are running it from within ftpsync, so it can change + # the way it works with mhop based on that. + RUNMIRRORARGS="${RUNMIRRORARGS} -f" + + if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then + # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will + # not get to this point. + # So if that happens, tell runmirrors we are doing mhop + RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" + elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" + elif [ "xtruex" = "x${SYNCALL}x" ]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -k all" + fi + log "Trigger slave mirrors" + ${BASEDIR}/bin/runmirrors "${RUNMIRRORARGS}" + log "Trigger slave done" + + HOOK=( + HOOKNR=5 + HOOKSCR=${HOOK5} + ) + hook $HOOK + fi fi # All done, lets call cleanup -- cgit v1.2.3 From ffd522df0efa4e480456fddfb3d26d7076c7cfd1 Mon Sep 17 00:00:00 2001 From: Uli Martens Date: Tue, 29 Dec 2009 18:23:14 +0100 Subject: quote some more arguments Signed-off-by: Joerg Jaspert --- bin/runmirrors | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/runmirrors b/bin/runmirrors index 6952228..92dda78 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -55,10 +55,10 @@ if [ $# -gt 0 ]; then # Yeah well, new style, starting with - for getopts while getopts ':a:k:fh' OPTION ; do case $OPTION in - a) CONF=${OPTARG} - NAME=${NAME}-${CONF} + a) CONF="${OPTARG}" + NAME="${NAME}-${CONF}" ;; - k) PUSHKIND=${OPTARG} + k) PUSHKIND="${OPTARG}" ;; f) FROMFTPSYNC="true" ;; -- cgit v1.2.3 From fa36383caf8b29facb367d291062c15951763c7b Mon Sep 17 00:00:00 2001 From: Uli Martens Date: Tue, 29 Dec 2009 18:25:56 +0100 Subject: fix whitespace indenting to minimize diff Signed-off-by: Joerg Jaspert --- bin/runmirrors | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bin/runmirrors b/bin/runmirrors index 92dda78..6fe3d0c 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -227,17 +227,17 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do # Built our array SIGNAL_OPTS=( - MIRROR="${MLNAME}" - HOSTNAME="${MHOSTNAME}" - USERNAME="${MUSER}" - SSHPROTO="${MPROTO}" - SSHKEY="${MKEYFILE}" - SSHOPTS="${SSHOPT/ /#}" - PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1" - PUSHTYPE="${MTYPE}" - PUSHARCHIVE=${PUSHARCHIVE} - PUSHKIND=${PUSHKIND} - FROMFTPSYNC=${FROMFTPSYNC} + MIRROR="${MLNAME}" + HOSTNAME="${MHOSTNAME}" + USERNAME="${MUSER}" + SSHPROTO="${MPROTO}" + SSHKEY="${MKEYFILE}" + SSHOPTS="${SSHOPT/ /#}" + PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1" + PUSHTYPE="${MTYPE}" + PUSHARCHIVE=${PUSHARCHIVE} + PUSHKIND=${PUSHKIND} + FROMFTPSYNC=${FROMFTPSYNC} ) # And finally, push the mirror -- cgit v1.2.3 From f5aba53bbbb602693a2ece17cb271be507c3c5fd Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Wed, 30 Dec 2009 21:24:18 +0100 Subject: ftpsync tracefile only update the tracefile if it is a stage2 or all sync Signed-off-by: Joerg Jaspert --- bin/ftpsync | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index d016f89..ce77a21 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -448,10 +448,14 @@ function mainroutine() { hook $HOOK done - if [ -d "`dirname "${TO}/${TRACE}"`" ]; then - LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" - echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" - echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" + # We only update our tracefile when we had a stage2 or an all sync. + # Otherwise we would update it after stage1 already, which is wrong. + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then + if [ -d "`dirname "${TO}/${TRACE}"`" ]; then + LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" + echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" + echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" + fi fi HOOK=( -- cgit v1.2.3 From 7b011505f809740d5dad18f056723d2d9166bdc9 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Wed, 30 Dec 2009 21:24:43 +0100 Subject: runmirrors update copyright Signed-off-by: Joerg Jaspert --- bin/runmirrors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/runmirrors b/bin/runmirrors index 6fe3d0c..a1623fd 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -7,7 +7,7 @@ set -u # Based losely on existing scripts, written by an unknown number of # different people over the years. # -# Copyright (C) 2008 Joerg Jaspert +# Copyright (C) 2008, 2009 Joerg Jaspert # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as -- cgit v1.2.3 From 24de4a2f1d314be85bb3eb14c15bd0bf8de015ae Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Wed, 30 Dec 2009 21:25:16 +0100 Subject: common treat mhop mirror entries same as staged in this elif Signed-off-by: Joerg Jaspert --- etc/common | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/common b/etc/common index a45483c..2f6af01 100644 --- a/etc/common +++ b/etc/common @@ -76,7 +76,6 @@ signal () { date -u >> "${LOGDIR}/${MIRROR}.log" PUSHARGS="" - # Archive is non-empty, so tell the downstreams if [ -n "${PUSHARCHIVE}" ]; then PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" @@ -95,7 +94,7 @@ signal () { echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log" PUSHARGS1="sync:all" ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 & - elif [ "xstagedx" = "x${PUSHTYPE}x" ]; then + elif [ "xstagedx" = "x${PUSHTYPE}x" ] || [ "xmhopx" = "x${PUSHTYPE}x" ]; then # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log" -- cgit v1.2.3 From bce457d4e1f83d2ce0a80a889cb6462199387c1a Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Thu, 31 Dec 2009 15:11:25 +0100 Subject: runmirrors only add -conf to the name when it has value Signed-off-by: Joerg Jaspert --- bin/runmirrors | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/runmirrors b/bin/runmirrors index a1623fd..6d4faed 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -56,7 +56,9 @@ if [ $# -gt 0 ]; then while getopts ':a:k:fh' OPTION ; do case $OPTION in a) CONF="${OPTARG}" - NAME="${NAME}-${CONF}" + if [ -n "${CONF}" ]; then + NAME="${NAME}-${CONF}" + fi ;; k) PUSHKIND="${OPTARG}" ;; -- cgit v1.2.3 From 52438f4293c217998f359f617f3055d3039ea63b Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Thu, 31 Dec 2009 15:24:40 +0100 Subject: ftpsync only set -a argument for runmirrors when it actually has one Signed-off-by: Joerg Jaspert --- bin/ftpsync | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index ce77a21..1af9348 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -476,8 +476,11 @@ function mainroutine() { if [ x${HUB} = "xtrue" ]; then # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then - # We tell runmirrors about the archive we are running on. Might be empty. - RUNMIRRORARGS="-a ${ARCHIVE}" + RUNMIRRORARGS="" + if [ -n "${ARCHIVE}" ]; then + # We tell runmirrors about the archive we are running on. + RUNMIRRORARGS="-a ${ARCHIVE}" + fi # We also tell runmirrors that we are running it from within ftpsync, so it can change # the way it works with mhop based on that. RUNMIRRORARGS="${RUNMIRRORARGS} -f" -- cgit v1.2.3 From 464b2942c49d25cf9cbc8f76becfa155981bd86b Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Thu, 31 Dec 2009 15:44:23 +0100 Subject: runmirrors default is mhop and use && when checking for things not enabled Signed-off-by: Joerg Jaspert --- bin/runmirrors | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/runmirrors b/bin/runmirrors index 6d4faed..275f82e 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -37,7 +37,7 @@ Usage:\n\n 2.) using getopt style parameters:\n \t -a [NAME] - Same as 1.) above, used for the config files. Default empty.\n -\t -k [TYPE] - Type of push. all, stage2, mhop. Default all.\n +\t -k [TYPE] - Type of push. all, stage2, mhop. Default mhop.\n \t -f - Run from within the mirrorscript ftpsync. Don't use from commandline!\n \t -h - Print this help and exit " @@ -83,8 +83,8 @@ CONF=${CONF:-""} # Set the default to all, if we didnt get told about it. Currently # valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1, -# stage2 - staged push, second phase. Default is all. -PUSHKIND=${PUSHKIND:-"all"} +# stage2 - staged push, second phase. Default is mhop. +PUSHKIND=${PUSHKIND:-"mhop"} # If we are pushed from within ftpsync. Default false. FROMFTPSYNC=${FROMFTPSYNC:-"false"} @@ -184,7 +184,7 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do # If we are told we have a mhop sync to do, we will only look at staged/mhop # entries and ignore the rest. if [ "x${PUSHKIND}x" = "xmhopx" ]; then - if [ "x${MTYPE}x" != "xstagedx" ] || [ "x${MTYPE}x" != "xmhopx" ]; then + if [ "x${MTYPE}x" != "xstagedx" ] && [ "x${MTYPE}x" != "xmhopx" ]; then continue fi fi -- cgit v1.2.3 From c7ecee0e9700f796ab63388a60095c478952eb82 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Thu, 31 Dec 2009 16:26:36 +0100 Subject: ftpsync, runmirrors fixup calling of runmirrors log more what we are supposed to do Signed-off-by: Joerg Jaspert --- bin/ftpsync | 2 +- bin/runmirrors | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ftpsync b/bin/ftpsync index 1af9348..98f80d4 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -496,7 +496,7 @@ function mainroutine() { RUNMIRRORARGS="${RUNMIRRORARGS} -k all" fi log "Trigger slave mirrors" - ${BASEDIR}/bin/runmirrors "${RUNMIRRORARGS}" + ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS} log "Trigger slave done" HOOK=( diff --git a/bin/runmirrors b/bin/runmirrors index 275f82e..e320722 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -148,7 +148,7 @@ mkdir -p "${LOCKDIR}" trap 'log "Mirrorpush done" >> ${LOG}; savelog "${LOG}" > /dev/null' EXIT -log "Pushing leaf mirrors" >> ${LOG} +log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> ${LOG} HOOK=( HOOKNR=1 -- cgit v1.2.3 From f190bba422623faf981d793830867e16c6c6fe97 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Thu, 31 Dec 2009 16:36:51 +0100 Subject: runmirrors only skip non-mhop/staged entries if we are called from within ftpsync. When we are called outside, ie. starting the mirror run, we do have to push everything and can not skip. Signed-off-by: Joerg Jaspert --- bin/runmirrors | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/runmirrors b/bin/runmirrors index e320722..752e351 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -181,9 +181,9 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do continue fi - # If we are told we have a mhop sync to do, we will only look at staged/mhop - # entries and ignore the rest. - if [ "x${PUSHKIND}x" = "xmhopx" ]; then + # If we are told we have a mhop sync to do and are called from within ftpsync, + # we will only look at staged/mhop entries and ignore the rest. + if [ "x${PUSHKIND}x" = "xmhopx" ] && [ "x${FROMFTPSYNC}x" = "xtruex" ]; then if [ "x${MTYPE}x" != "xstagedx" ] && [ "x${MTYPE}x" != "xmhopx" ]; then continue fi -- cgit v1.2.3