diff options
| -rwxr-xr-x | bin/runmirrors | 39 | ||||
| -rw-r--r-- | etc/common | 94 | ||||
| -rw-r--r-- | etc/runmirrors.mirror.sample | 17 |
3 files changed, 122 insertions, 28 deletions
diff --git a/bin/runmirrors b/bin/runmirrors index 50014da..06423be 100755 --- a/bin/runmirrors +++ b/bin/runmirrors @@ -1,11 +1,14 @@ #! /bin/bash set -e +set -u # runmirrors script for Debian # Based losely on existing scripts, written by an unknown number of # different people over the years. # +# Copyright (C) 2008 Joerg Jaspert <joerg@debian.org> +# # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; version 2. @@ -44,6 +47,8 @@ export HOME LOGNAME USER PATH BASEDIR LOGDIR=${LOGDIR:-"${BASEDIR}/log"} # Our own logfile LOG=${LOG:-"${LOGDIR}/${NAME}.log"} +# Our lockfile directory +LOCKDIR=${LOCKDIR:-"${BASEDIR}/locks"} # How to rotate the log SAVELOG=${SAVELOG:-"savelog -t -c 14"} # Amount of delay between mirrors if the mirror file contains a DELAY @@ -55,13 +60,14 @@ PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} # extra ssh options we might want hostwide SSH_OPTS=${SSH_OPTS:-""} -# Make sure we have our log directory -mkdir -p ${LOGDIR} - # Some sane defaults cd ${BASEDIR} umask 002 +# Make sure we have our log and lock directories +mkdir -p ${LOGDIR} +mkdir -p ${LOCKDIR} + trap 'log "Mirrorpush done" >> ${LOG}; ${SAVELOG} "${LOG}" > /dev/null' EXIT log "Pushing leaf mirrors" >> ${LOG} @@ -74,14 +80,21 @@ if [ -n "${HOOK1}" ]; then fi # From here on we do *NOT* want to exit on errors. We don't want to -# stop pushing mirrors just because we can't reach one atm. +# stop pushing mirrors just because we can't reach one of them. set +e +# Built up our list of 2-stage mirrors. +PUSHLOCKS="" +egrep '^staged' ${MIRRORS} | +while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do + PUSHLOCKS="${PUSHLOCKS} ${LOCKDIR}/${MLNAME}.stage1" +done + # Now read our mirrorfile and push the mirrors defined in there. # We use grep to easily sort out all lines having a # in front of them or are empty. egrep -v '^[[:space:]]*(#|$)' ${MIRRORS} | -while read MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do - if [ "x${MLNAME}x" = "xDELAYx" ]; then +while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do + if [ "x${MTYPE}x" = "xDELAYx" ]; then # We should wait a bit. log "Delay of ${DELAY} seconds requested, sleeping" >> ${LOG} sleep ${DELAY} @@ -102,10 +115,22 @@ while read MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do continue fi + # Built our array + SIGNAL_OPTS=( + MIRROR="${MLNAME}" + HOSTNAME="${MHOSTNAME}" + USERNAME="${MUSER}" + SSHPROTO="${MPROTO}" + SSHKEY="${MKEYFILE}" + PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1" + PUSHTYPE="${MTYPE}" + ) + # And finally, push the mirror log "Trigger ${MLNAME}" >> ${LOG} - signal ${MLNAME} ${MHOSTNAME} ${MUSER} ${MPROTO} ${MKEYFILE} "${SSH_OPTS}" + signal "${SIGNAL_OPTS}" & log "Trigger for ${MLNAME} done" >> ${LOG} + if [ -n "${HOOK2}" ]; then log "Running hook2: ${HOOK2}" >> ${LOG} ${HOOK1} @@ -1,34 +1,92 @@ +# -*- mode:sh -*- # Little common functions # push a mirror attached to us. -# Arguments: -# $1 - Name for the mirror, also basename for the logfile -# $2 - Hostname to push to -# $3 - Username there -# $4 - Protocol version, either 1 or 2. -# $5 - the ssh private key file to use for this push -# $6 - any other option ssh accepts, passed blindly, be careful +# Arguments (using an array named SIGNAL_OPTS): +# +# $MIRROR - Name for the mirror, also basename for the logfile +# $HOSTNAME - Hostname to push to +# $USERNAME - Username there +# $SSHPROTO - Protocol version, either 1 or 2. +# $SSHKEY - the ssh private key file to use for this push +# $SSHOPTS - any other option ssh accepts, passed blindly, be careful +# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged +# $PUSHTYPE - what kind of push should be done? +# all - normal, just push once with ssh backgrounded and finish +# staged - staged. first push stage1, then wait for $PUSHLOCKs to appear, +# then push stage2 +# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) +# $PUSHCB - do we want a callback? # # This function assumes that the variable LOG is set to a directory where # logfiles can be written to. -# Pushes will be done in background. +# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings +# (list of "lock"files) to wait for if you want pushtype=staged +# +# Pushes might be done in background (for type all). signal () { - if [ $# -lt 5 ]; then - echo "Called with only $# parameters, expect at least 5" - return 2 - fi + ARGS="SIGNAL_OPTS[*]" + local ${!SIGNAL_OPTS} # Defaults we always want, no matter what SSH_OPTIONS="-o BatchMode=yes -o SetupTimeOut=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" - if [ $# -eq 6 ]; then - # The sixth sense^Wparameter, add it - SSH_OPTIONS="$SSH_OPTIONS $6" + if [ $SSHOPTS -n ]; then + SSH_OPTIONS="$SSH_OPTIONS $SSHOPTS" + fi + + if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ]; then + # Idiots, we only want 1 or 2. Cant decide? Lets force 2 + ${SSHPROTO}=2 + fi + + date -u >> ${LOGDIR}/${MIRROR}.log + + if [ "xallx" = "x${PUSHTYPE}x" ]; then + # Default normal "fire and forget" push + ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "sync:all" >>${LOGDIR}/${MIRROR}.log 2>&1 & + elif [ "xstagedx" = "x{$PUSHTYPE}x"]; then + # Want a staged push. Fine, lets do that + + PUSHARGS="" + if [ -n ${PUSHARCHIVE} ]; then + PUSHARGS="${PUSHARGS sync:archive:${PUSHARCHIVE}" + fi + if [ -n ${PUSHCB} ]; then + PUSHARGS="${PUSHARGS sync:callback" + fi + + # Step1: Do a push to only sync stage1, do not background + PUSHARGS1="sync:stage1" + ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>${LOGDIR}/${MIRROR}.log 2>&1 + touch ${PUSHLOCKOWN} + + # Step2: Wait for all the other "lock"files to appear. + tries=0 + # We do not wait forever + while [ $tries -lt 30 ]; do + total=0 + found=0 + for file in ${PUSHLOCKS}; do + total=$((total + 1)) + if [ -f ${file} ]; then + found=$(($found + 1)) + fi + done + if [ $total -eq $found ]; + break; + fi + sleep 20 + done + rm -f ${PUSHLOCKOWN} + + # Steo3: It either timed out or we have all the "lock"files, sync stage2 + PUSHARGS1="sync:stage2" + ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>${LOGDIR}/${MIRROR}.log 2>&1 + else + # Can't decide? Then you get nothing. fi - # Finally call ssh - date -u >> ${LOGDIR}/$1.log - ssh $SSH_OPTIONS -i "$5" -o"user $3" -$4 "$2" sleep 1 >>${LOGDIR}/$1.log 2>&1 & } # callback, used by ftpsync diff --git a/etc/runmirrors.mirror.sample b/etc/runmirrors.mirror.sample index 33f30c8..e529376 100644 --- a/etc/runmirrors.mirror.sample +++ b/etc/runmirrors.mirror.sample @@ -1,7 +1,12 @@ # Definition of mirror hosts we push. # One mirror per line, with the following fields defined. # -# ShortName HostName User SSHProtocol SSHKeyFile +# Type ShortName HostName User SSHProtocol SSHKeyFile +# +# Type is either all or staged, meaning: +# all - do a "normal" push. Trigger them, go on. +# staged - do a two-stage push, waiting for them after stage 2(and all others that +# are staged) before doing stage2 # # ShortName will be used as a shorthand in logfile outputs and for the logfile # where every ssh output gets redirected to. @@ -10,11 +15,17 @@ # If SSHProtocol is empty, it will default to 2, but if you want to # define a keyfile you HAVE TO set protocol too! # -# Example: -# eu.puccini puccini.debian.org archvsync 2 ~/.ssh/push_puccini +# Examples: +# all eu.puccini puccini.debian.org archvsync 2 ~/.ssh/push_puccini # # will push puccini.debian.org, user archvsync, using ssh protocol 2 and the specified ssh key. # +# staged eu.puccini puccini.debian.org +# staged eu.powell powell.debian.org +# +# will push both puccini and powell in stage1, waiting for both to +# finish stage1 before stage2 gets pushed. +# # One special value is allowed: DELAY # This word has to be on a line itself, nothing else, not even # whitespace. It will trigger a pause of $DELAY seconds between the two |