summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xbin/ftpsync411
-rwxr-xr-xbin/pushpdo58
-rwxr-xr-xbin/runmirrors94
-rwxr-xr-xbin/websync176
4 files changed, 369 insertions, 370 deletions
diff --git a/bin/ftpsync b/bin/ftpsync
index f70d506..6e9daeb 100755
--- a/bin/ftpsync
+++ b/bin/ftpsync
@@ -68,44 +68,44 @@ VERSION="80286"
# "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the
# trick.
check_commandline() {
- while [ $# -gt 0 ]; do
- case "$1" in
- sync:stage1)
- SYNCSTAGE1="true"
- SYNCALL="false"
- ;;
- sync:stage2)
- SYNCSTAGE2="true"
- SYNCALL="false"
- ;;
- sync:callback)
- SYNCCALLBACK="true"
- ;;
- sync:archive:*)
- ARCHIVE=${1##sync:archive:}
- # We do not like / or . in the remotely supplied archive name.
- ARCHIVE=${ARCHIVE//\/}
- ARCHIVE=${ARCHIVE//.}
- ;;
- sync:all)
- SYNCALL="true"
- ;;
- sync:mhop)
- SYNCMHOP="true"
- ;;
- *)
- echo "Unknown option ${1} ignored"
- ;;
- esac
- shift # Check next set of parameters.
- done
+ while [ $# -gt 0 ]; do
+ case "$1" in
+ sync:stage1)
+ SYNCSTAGE1="true"
+ SYNCALL="false"
+ ;;
+ sync:stage2)
+ SYNCSTAGE2="true"
+ SYNCALL="false"
+ ;;
+ sync:callback)
+ SYNCCALLBACK="true"
+ ;;
+ sync:archive:*)
+ ARCHIVE=${1##sync:archive:}
+ # We do not like / or . in the remotely supplied archive name.
+ ARCHIVE=${ARCHIVE//\/}
+ ARCHIVE=${ARCHIVE//.}
+ ;;
+ sync:all)
+ SYNCALL="true"
+ ;;
+ sync:mhop)
+ SYNCMHOP="true"
+ ;;
+ *)
+ echo "Unknown option ${1} ignored"
+ ;;
+ esac
+ shift # Check next set of parameters.
+ done
}
# All the stuff we want to do when we exit, no matter where
cleanup() {
- trap - ERR TERM HUP INT QUIT EXIT
- # all done. Mail the log, exit.
- log "Mirrorsync done";
+ trap - ERR TERM HUP INT QUIT EXIT
+ # all done. Mail the log, exit.
+ log "Mirrorsync done";
# Lets get a statistical value
SPEED=$(
@@ -121,47 +121,46 @@ cleanup() {
fi
log "Rsync transfer speed: ${SPEED} KB/s"
- if [ -n "${MAILTO}" ]; then
+ if [ -n "${MAILTO}" ]; then
# In case rsync had something on stderr
- if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
- mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
- fi
- if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
- # And the normal log
- MAILFILES="${LOG}"
- if [ "x${FULLLOGS}x" = "xtruex" ]; then
- # Someone wants full logs including rsync
- MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
- fi
- cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
- fi
- fi
-
- savelog "${LOGDIR}/rsync-${NAME}.log"
- savelog "${LOGDIR}/rsync-${NAME}.error"
- savelog "$LOG" > /dev/null
-
- rm -f "${LOCK}"
+ if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
+ mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
+ fi
+ if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
+ # And the normal log
+ MAILFILES="${LOG}"
+ if [ "x${FULLLOGS}x" = "xtruex" ]; then
+ # Someone wants full logs including rsync
+ MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
+ fi
+ cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
+ fi
+ fi
+
+ savelog "${LOGDIR}/rsync-${NAME}.log"
+ savelog "${LOGDIR}/rsync-${NAME}.error"
+ savelog "$LOG" > /dev/null
+
+ rm -f "${LOCK}"
}
# Check rsyncs return value
check_rsync() {
-
- ret=$1
- msg=$2
-
- # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
- # and us re-running. If it's not, uplink is broken anyways.
- case "${ret}" in
- 0) return 0;;
- 24) return 0;;
- 23) return 2;;
- 30) return 2;;
- *)
- error "ERROR: ${msg}"
- return 1
- ;;
- esac
+ ret=$1
+ msg=$2
+
+ # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
+ # and us re-running. If it's not, uplink is broken anyways.
+ case "${ret}" in
+ 0) return 0;;
+ 24) return 0;;
+ 23) return 2;;
+ 30) return 2;;
+ *)
+ error "ERROR: ${msg}"
+ return 1
+ ;;
+ esac
}
########################################################################
@@ -181,37 +180,37 @@ SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""}
# Now, check if we got told about stuff via ssh
if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then
# We deliberately add "nothing" and ignore it right again, to avoid
- # people from outside putting some set options in the first place,
- # making us parse them...
- set "nothing" "${SSH_ORIGINAL_COMMAND}"
- shift
- # Yes, unqouted $* here. Or the function will only see it as one
- # parameter, which doesnt help the case in it.
- check_commandline $*
+ # people from outside putting some set options in the first place,
+ # making us parse them...
+ set "nothing" "${SSH_ORIGINAL_COMMAND}"
+ shift
+ # Yes, unqouted $* here. Or the function will only see it as one
+ # parameter, which doesnt help the case in it.
+ check_commandline $*
fi
# Now, we can locally override all the above variables by just putting
# them into the .ssh/authorized_keys file forced command.
if [ -n "${ORIGINAL_COMMAND}" ]; then
- set ${ORIGINAL_COMMAND}
- check_commandline $*
+ set ${ORIGINAL_COMMAND}
+ check_commandline $*
fi
# If we have been told to do stuff for a different archive than default,
# set the name accordingly.
ARCHIVE=${ARCHIVE:-""}
if [ -n "${ARCHIVE}" ]; then
- NAME="${NAME}-${ARCHIVE}"
+ NAME="${NAME}-${ARCHIVE}"
fi
# Now source the config for the archive we run on.
# (Yes, people can also overwrite the options above in the config file
# if they want to)
if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then
- . "${BASEDIR}/etc/${NAME}.conf"
+ . "${BASEDIR}/etc/${NAME}.conf"
else
- echo "Nono, you can't tell us about random archives. Bad boy!"
- exit 1
+ echo "Nono, you can't tell us about random archives. Bad boy!"
+ exit 1
fi
########################################################################
@@ -250,11 +249,11 @@ PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
# Where to send mails about mirroring to?
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
- # We are not on a debian.org host
- MAILTO=${MAILTO:-"root"}
+ # We are not on a debian.org host
+ MAILTO=${MAILTO:-"root"}
else
- # Yay, on a .debian.org host
- MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
+ # Yay, on a .debian.org host
+ MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
# Want errors only or every log?
ERRORSONLY=${ERRORSONLY:-"true"}
@@ -330,10 +329,10 @@ ARCH_EXCLUDE=${ARCH_EXCLUDE:-""}
for ARCH in ${ARCH_EXCLUDE}; do
EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude disks-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes"
if [ "${ARCH}" = "source" ]; then
- if [ -z ${SOURCE_EXCLUDE} ]; then
- SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc "
- fi
- fi
+ if [ -z ${SOURCE_EXCLUDE} ]; then
+ SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc "
+ fi
+ fi
done
# Hooks
@@ -370,29 +369,29 @@ function mainroutine() {
log "Mirrorsync start"
PUSHFROM="${SSH_CONNECTION%%\ *}"
if [ -n "${PUSHFROM}" ]; then
- log "We got pushed from ${PUSHFROM}"
+ log "We got pushed from ${PUSHFROM}"
fi
log "Acquired main lock"
if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
- if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then
- SYNCCALLBACK="false"
- error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
- fi
+ if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then
+ SYNCCALLBACK="false"
+ error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
+ fi
fi
HOOK=(
- HOOKNR=1
- HOOKSCR=${HOOK1}
+ HOOKNR=1
+ HOOKSCR=${HOOK1}
)
hook $HOOK
# Now, we might want to sync from anonymous too.
# This is that deep in this script so hook1 could, if wanted, change things!
if [ -z ${RSYNC_USER} ]; then
- RSYNCPTH="${RSYNC_HOST}"
+ RSYNCPTH="${RSYNC_HOST}"
else
- RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
+ RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
fi
# Now do the actual mirroring, and run as long as we have an updaterequired file.
@@ -400,137 +399,137 @@ function mainroutine() {
export RSYNC_PROXY
while [ -e "${UPDATEREQUIRED}" ]; do
- log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
+ log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
# if we want stage1 *or* all
- if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
- while [ -e "${UPDATEREQUIRED}" ]; do
- rm -f "${UPDATEREQUIRED}"
- log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
-
- set +e
- # Step one, sync everything except Packages/Releases
- ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
- ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
- result=$?
- set -e
-
- log "Back from rsync with returncode ${result}"
- done
- else
- # Fake a good resultcode
- result=0
- fi # Sync stage 1?
- rm -f "${UPDATEREQUIRED}"
-
- set +e
- check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
- GO=$?
- set -e
- if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
- log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
- elif [ ${GO} -ne 0 ]; then
- exit 3
- fi
-
- HOOK=(
- HOOKNR=2
- HOOKSCR=${HOOK2}
- )
- hook $HOOK
-
- # if we want stage2 *or* all
- if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
- log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
-
- set +e
- # We are lucky, it worked. Now do step 2 and sync again, this time including
- # the packages/releases files
- ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
- ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error
- result=$?
- set -e
-
- log "Back from rsync with returncode ${result}"
- else
- # Fake a good resultcode
- result=0
- fi # Sync stage 2?
-
- set +e
- check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
- GO=$?
- set -e
- if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
- log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
- elif [ ${GO} -ne 0 ]; then
- exit 4
- fi
-
- HOOK=(
- HOOKNR=3
- HOOKSCR=${HOOK3}
- )
- hook $HOOK
+ if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+ while [ -e "${UPDATEREQUIRED}" ]; do
+ rm -f "${UPDATEREQUIRED}"
+ log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+
+ set +e
+ # Step one, sync everything except Packages/Releases
+ ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+ ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
+ result=$?
+ set -e
+
+ log "Back from rsync with returncode ${result}"
+ done
+ else
+ # Fake a good resultcode
+ result=0
+ fi # Sync stage 1?
+ rm -f "${UPDATEREQUIRED}"
+
+ set +e
+ check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
+ GO=$?
+ set -e
+ if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
+ log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
+ elif [ ${GO} -ne 0 ]; then
+ exit 3
+ fi
+
+ HOOK=(
+ HOOKNR=2
+ HOOKSCR=${HOOK2}
+ )
+ hook $HOOK
+
+ # if we want stage2 *or* all
+ if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+ log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+
+ set +e
+ # We are lucky, it worked. Now do step 2 and sync again, this time including
+ # the packages/releases files
+ ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+ ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error
+ result=$?
+ set -e
+
+ log "Back from rsync with returncode ${result}"
+ else
+ # Fake a good resultcode
+ result=0
+ fi # Sync stage 2?
+
+ set +e
+ check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
+ GO=$?
+ set -e
+ if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
+ log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
+ elif [ ${GO} -ne 0 ]; then
+ exit 4
+ fi
+
+ HOOK=(
+ HOOKNR=3
+ HOOKSCR=${HOOK3}
+ )
+ hook $HOOK
done
# We only update our tracefile when we had a stage2 or an all sync.
# Otherwise we would update it after stage1 already, which is wrong.
- if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+ if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
if [ -d "`dirname "${TO}/${TRACE}"`" ]; then
LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
- echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
+ echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
fi
fi
HOOK=(
- HOOKNR=4
- HOOKSCR=${HOOK4}
+ HOOKNR=4
+ HOOKSCR=${HOOK4}
)
hook $HOOK
if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
- set +e
- callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
- set -e
+ set +e
+ callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
+ set -e
fi
# Remove the Archive-Update-in-Progress file before we push our downstreams.
- rm -f "${LOCK}"
+ rm -f "${LOCK}"
if [ x${HUB} = "xtrue" ]; then
- # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
- if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then
+ # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
+ if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then
RUNMIRRORARGS=""
if [ -n "${ARCHIVE}" ]; then
# We tell runmirrors about the archive we are running on.
- RUNMIRRORARGS="-a ${ARCHIVE}"
+ RUNMIRRORARGS="-a ${ARCHIVE}"
fi
- # We also tell runmirrors that we are running it from within ftpsync, so it can change
- # the way it works with mhop based on that.
- RUNMIRRORARGS="${RUNMIRRORARGS} -f"
-
- if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then
- # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
- # not get to this point.
- # So if that happens, tell runmirrors we are doing mhop
- RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
- elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then
- RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
- elif [ "xtruex" = "x${SYNCALL}x" ]; then
- RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
- fi
- log "Trigger slave mirrors using ${RUNMIRRORARGS}"
- ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS}
- log "Trigger slave done"
-
- HOOK=(
- HOOKNR=5
- HOOKSCR=${HOOK5}
- )
- hook $HOOK
- fi
+ # We also tell runmirrors that we are running it from within ftpsync, so it can change
+ # the way it works with mhop based on that.
+ RUNMIRRORARGS="${RUNMIRRORARGS} -f"
+
+ if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then
+ # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
+ # not get to this point.
+ # So if that happens, tell runmirrors we are doing mhop
+ RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
+ elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then
+ RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
+ elif [ "xtruex" = "x${SYNCALL}x" ]; then
+ RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
+ fi
+ log "Trigger slave mirrors using ${RUNMIRRORARGS}"
+ ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS}
+ log "Trigger slave done"
+
+ HOOK=(
+ HOOKNR=5
+ HOOKSCR=${HOOK5}
+ )
+ hook $HOOK
+ fi
fi
# All done, lets call cleanup
@@ -575,7 +574,7 @@ if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
unixtime=$(date +%s)
difference=$(( $unixtime - $stamptime ))
if [ ${difference} -ge ${LOCKTIMEOUT} ]; then
- # Took longer than 60 minutes? Assume it broke and take the lock
+ # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock
echo "$$" > "${LOCK}"
else
echo "Unable to start rsync, lock file younger than one hour"
diff --git a/bin/pushpdo b/bin/pushpdo
index 12512b5..7dfc440 100755
--- a/bin/pushpdo
+++ b/bin/pushpdo
@@ -52,16 +52,16 @@ KEYFILE=${KEYFILE:-".ssh/pushpackages"}
MIRRORPATH=${MIRRORPATH:-"/org/packages.debian.org/mirror/"}
# where to send mails to
if [ "x$(hostname -s)x" != "x${MIRRORNAME%%.debian.org}x" ]; then
- # We are not on a debian.org host
- MAILTO=${MAILTO:-"root"}
+ # We are not on a debian.org host
+ MAILTO=${MAILTO:-"root"}
else
- # Yay, on a .debian.org host
- MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
+ # Yay, on a .debian.org host
+ MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
- error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG}
- exit 5
+ error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG}
+ exit 5
fi
# Some sane defaults
@@ -84,29 +84,29 @@ set +e
# We use grep to easily sort out all lines having a # in front of them or are empty.
egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
while read MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do
- # Process the two options that can be left blank in the config
- if [ -z ${MPROTO} ]; then
- MPROTO=2
- fi
- if [ -z ${MKEYFILE} ]; then
- MKEYFILE="${BASEDIR}/${KEYFILE}"
- fi
- # Now, people can do stupid things and leave out the protocol, but
- # define a keyfile...
- if [ ${MPROTO} -ne 1 ] && [ ${MPROTO} -ne 2 ]; then
- error "Need a correct ssh protocol version for ${MLNAME}, skipping" >> ${LOG}
- continue
- fi
-
- # And finally, push the mirror
- log "Pushing ${MLNAME}" >> ${LOG}
- # This needs a limited ssh key on the other side, something like
- # no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="rsync --server -vlogDtpr . /srv/mirrors/packages.debian.org/",from="87.106.64.223,2001:8d8:80:11::35d,powell.debian.org" ssh-rsa.....
- rsync -e "ssh -i ${MKEYFILE} -${MPROTO}" -av --stats "${MIRRORPATH}" ${MUSER}@${MHOSTNAME}:/does/not/matter >"${LOGDIR}/${MLNAME}.log"
- log "Pushing ${MLNAME} done" >> ${LOG}
- savelog ${LOGDIR}${MLNAME}.log
-
- set +e
+ # Process the two options that can be left blank in the config
+ if [ -z ${MPROTO} ]; then
+ MPROTO=2
+ fi
+ if [ -z ${MKEYFILE} ]; then
+ MKEYFILE="${BASEDIR}/${KEYFILE}"
+ fi
+ # Now, people can do stupid things and leave out the protocol, but
+ # define a keyfile...
+ if [ ${MPROTO} -ne 1 ] && [ ${MPROTO} -ne 2 ]; then
+ error "Need a correct ssh protocol version for ${MLNAME}, skipping" >> ${LOG}
+ continue
+ fi
+
+ # And finally, push the mirror
+ log "Pushing ${MLNAME}" >> ${LOG}
+ # This needs a limited ssh key on the other side, something like
+ # no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="rsync --server -vlogDtpr . /srv/mirrors/packages.debian.org/",from="87.106.64.223,2001:8d8:80:11::35d,powell.debian.org" ssh-rsa.....
+ rsync -e "ssh -i ${MKEYFILE} -${MPROTO}" -av --stats "${MIRRORPATH}" ${MUSER}@${MHOSTNAME}:/does/not/matter >"${LOGDIR}/${MLNAME}.log"
+ log "Pushing ${MLNAME} done" >> ${LOG}
+ savelog ${LOGDIR}${MLNAME}.log
+
+ set +e
done
exit 0
diff --git a/bin/runmirrors b/bin/runmirrors
index 5b04ecb..1b4436b 100755
--- a/bin/runmirrors
+++ b/bin/runmirrors
@@ -69,7 +69,7 @@ if [ $# -gt 0 ]; then
;;
*) echo "Invalid usage"
- echo -e $HELP
+ echo -e $HELP
exit 1
;;
esac
@@ -119,16 +119,16 @@ PUSHDELAY=${PUSHDELAY:-600}
KEYFILE=${KEYFILE:-".ssh/pushmirror"}
# where to send mails to
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
- # We are not on a debian.org host
- MAILTO=${MAILTO:-"root"}
+ # We are not on a debian.org host
+ MAILTO=${MAILTO:-"root"}
else
- # Yay, on a .debian.org host
- MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
+ # Yay, on a .debian.org host
+ MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
- error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG}
- exit 5
+ error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG}
+ exit 5
fi
# Hooks
@@ -151,8 +151,8 @@ trap 'log "Mirrorpush done" >> ${LOG}; savelog "${LOG}" > /dev/null' EXIT
log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> ${LOG}
HOOK=(
- HOOKNR=1
- HOOKSCR=${HOOK1}
+ HOOKNR=1
+ HOOKSCR=${HOOK1}
)
hook $HOOK
@@ -171,15 +171,15 @@ rm -f "${LOCKDIR}/all_stage1"
# We use grep to easily sort out all lines having a # in front of them or are empty.
egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do
- if [ "x${MTYPE}x" = "xDELAYx" ]; then
- # We should wait a bit.
- if [ -z ${MLNAME} ]; then
- MLNAME=600
- fi
- log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}"
- sleep ${MLNAME}
- continue
- fi
+ if [ "x${MTYPE}x" = "xDELAYx" ]; then
+ # We should wait a bit.
+ if [ -z ${MLNAME} ]; then
+ MLNAME=600
+ fi
+ log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}"
+ sleep ${MLNAME}
+ continue
+ fi
# If we are told we have a mhop sync to do and are called from within ftpsync,
# we will only look at staged/mhop entries and ignore the rest.
@@ -223,36 +223,36 @@ while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do
fi
else
MPROTO=2
- MKEYFILE="${BASEDIR}/${KEYFILE}"
+ MKEYFILE="${BASEDIR}/${KEYFILE}"
SSHOPT=""
fi
- # Built our array
- SIGNAL_OPTS=(
- MIRROR="${MLNAME}"
- HOSTNAME="${MHOSTNAME}"
- USERNAME="${MUSER}"
- SSHPROTO="${MPROTO}"
- SSHKEY="${MKEYFILE}"
- SSHOPTS="${SSHOPT/ /#}"
- PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1"
- PUSHTYPE="${MTYPE}"
- PUSHARCHIVE=${PUSHARCHIVE}
- PUSHKIND=${PUSHKIND}
- FROMFTPSYNC=${FROMFTPSYNC}
- )
-
- # And finally, push the mirror
- log "Trigger ${MLNAME}" >> ${LOG}
- signal "${SIGNAL_OPTS}" &
- log "Trigger for ${MLNAME} done" >> ${LOG}
-
- HOOK=(
- HOOKNR=2
- HOOKSCR=${HOOK2}
- )
- hook $HOOK
- set +e
+ # Built our array
+ SIGNAL_OPTS=(
+ MIRROR="${MLNAME}"
+ HOSTNAME="${MHOSTNAME}"
+ USERNAME="${MUSER}"
+ SSHPROTO="${MPROTO}"
+ SSHKEY="${MKEYFILE}"
+ SSHOPTS="${SSHOPT/ /#}"
+ PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1"
+ PUSHTYPE="${MTYPE}"
+ PUSHARCHIVE=${PUSHARCHIVE}
+ PUSHKIND=${PUSHKIND}
+ FROMFTPSYNC=${FROMFTPSYNC}
+ )
+
+ # And finally, push the mirror
+ log "Trigger ${MLNAME}" >> ${LOG}
+ signal "${SIGNAL_OPTS}" &
+ log "Trigger for ${MLNAME} done" >> ${LOG}
+
+ HOOK=(
+ HOOKNR=2
+ HOOKSCR=${HOOK2}
+ )
+ hook $HOOK
+ set +e
done
# If we are run from within ftpsync *and* have an mhop push to send on, we have
@@ -278,8 +278,8 @@ if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then
fi
HOOK=(
- HOOKNR=3
- HOOKSCR=${HOOK3}
+ HOOKNR=3
+ HOOKSCR=${HOOK3}
)
hook $HOOK
diff --git a/bin/websync b/bin/websync
index bcb2e52..b96ba54 100755
--- a/bin/websync
+++ b/bin/websync
@@ -48,51 +48,51 @@ VERSION="0815"
########################################################################
# All the stuff we want to do when we exit, no matter where
cleanup() {
- trap - ERR TERM HUP INT QUIT EXIT
- # all done. Mail the log, exit.
- log "Mirrorsync done";
- if [ -n "${MAILTO}" ]; then
+ trap - ERR TERM HUP INT QUIT EXIT
+ # all done. Mail the log, exit.
+ log "Mirrorsync done";
+ if [ -n "${MAILTO}" ]; then
# In case rsync had something on stderr
- if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
- mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
- fi
- if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
- # And the normal log
- MAILFILES="${LOG}"
- if [ "x${FULLLOGS}x" = "xtruex" ]; then
- # Someone wants full logs including rsync
- MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
- fi
- cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
- fi
- fi
-
- savelog "${LOGDIR}/rsync-${NAME}.log"
- savelog "${LOGDIR}/rsync-${NAME}.error"
- savelog "$LOG" > /dev/null
-
- rm -f "${LOCK}"
+ if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
+ mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
+ fi
+ if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
+ # And the normal log
+ MAILFILES="${LOG}"
+ if [ "x${FULLLOGS}x" = "xtruex" ]; then
+ # Someone wants full logs including rsync
+ MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
+ fi
+ cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
+ fi
+ fi
+
+ savelog "${LOGDIR}/rsync-${NAME}.log"
+ savelog "${LOGDIR}/rsync-${NAME}.error"
+ savelog "$LOG" > /dev/null
+
+ rm -f "${LOCK}"
}
# Check rsyncs return value
check_rsync() {
- ret=$1
- msg=$2
-
- # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
- # and us re-running. If it's not, uplink is broken anyways.
- case "${ret}" in
- 0) return 0;;
- 24) return 0;;
- 23) return 2;;
- 30) return 2;;
- *)
- error "ERROR: ${msg}"
- return 1
- ;;
- esac
+ ret=$1
+ msg=$2
+
+ # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
+ # and us re-running. If it's not, uplink is broken anyways.
+ case "${ret}" in
+ 0) return 0;;
+ 24) return 0;;
+ 23) return 2;;
+ 30) return 2;;
+ *)
+ error "ERROR: ${msg}"
+ return 1
+ ;;
+ esac
}
########################################################################
@@ -127,11 +127,11 @@ PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
# Where to send mails about mirroring to?
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
- # We are not on a debian.org host
- MAILTO=${MAILTO:-"root"}
+ # We are not on a debian.org host
+ MAILTO=${MAILTO:-"root"}
else
- # Yay, on a .debian.org host
- MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
+ # Yay, on a .debian.org host
+ MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
# Want errors only or every log?
ERRORSONLY=${ERRORSONLY:-"true"}
@@ -205,13 +205,13 @@ touch "${UPDATEREQUIRED}"
# Check to see if another sync is in progress
if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
- if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
- # Process does either not exist or is not owned by us.
- echo "$$" > "${LOCK}"
- else
- echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
- exit 1
- fi
+ if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
+ # Process does either not exist or is not owned by us.
+ echo "$$" > "${LOCK}"
+ else
+ echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
+ exit 1
+ fi
fi
trap cleanup EXIT ERR TERM HUP INT QUIT
@@ -223,22 +223,22 @@ exec >"$LOG" 2>&1 </dev/null
log "Mirrorsync start"
PUSHFROM="${SSH_CONNECTION%%\ *}"
if [ -n "${PUSHFROM}" ]; then
- log "We got pushed from ${PUSHFROM}"
+ log "We got pushed from ${PUSHFROM}"
fi
log "Acquired main lock"
HOOK=(
- HOOKNR=1
- HOOKSCR=${HOOK1}
+ HOOKNR=1
+ HOOKSCR=${HOOK1}
)
hook $HOOK
# Now, we might want to sync from anonymous too.
# This is that deep in this script so hook1 could, if wanted, change things!
if [ -z ${RSYNC_USER} ]; then
- RSYNCPTH="${RSYNC_HOST}"
+ RSYNCPTH="${RSYNC_HOST}"
else
- RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
+ RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
fi
# Now do the actual mirroring, and run as long as we have an updaterequired file.
@@ -246,35 +246,35 @@ export RSYNC_PASSWORD
export RSYNC_PROXY
while [ -e "${UPDATEREQUIRED}" ]; do
- log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
+ log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
- rm -f "${UPDATEREQUIRED}"
- log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+ rm -f "${UPDATEREQUIRED}"
+ log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
- set +e
- ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \
- ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
- result=$?
- set -e
+ set +e
+ ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \
+ ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
+ result=$?
+ set -e
- log "Back from rsync with returncode ${result}"
+ log "Back from rsync with returncode ${result}"
- set +e
- check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}"
- GO=$?
- set -e
+ set +e
+ check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}"
+ GO=$?
+ set -e
- if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
- log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
- elif [ ${GO} -ne 0 ]; then
- exit 3
- fi
+ if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
+ log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
+ elif [ ${GO} -ne 0 ]; then
+ exit 3
+ fi
- HOOK=(
- HOOKNR=2
- HOOKSCR=${HOOK2}
- )
- hook $HOOK
+ HOOK=(
+ HOOKNR=2
+ HOOKSCR=${HOOK2}
+ )
+ hook $HOOK
done
@@ -284,21 +284,21 @@ echo "Used websync version: ${VERSION}" >> "${TO}/${TRACE}"
echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
HOOK=(
- HOOKNR=3
- HOOKSCR=${HOOK3}
+ HOOKNR=3
+ HOOKSCR=${HOOK3}
)
hook $HOOK
if [ x${HUB} = "xtrue" ]; then
- log "Trigger slave mirrors"
- ${BASEDIR}/bin/runmirrors "websync"
- log "Trigger slave done"
-
- HOOK=(
- HOOKNR=4
- HOOKSCR=${HOOK4}
- )
- hook $HOOK
+ log "Trigger slave mirrors"
+ ${BASEDIR}/bin/runmirrors "websync"
+ log "Trigger slave done"
+
+ HOOK=(
+ HOOKNR=4
+ HOOKSCR=${HOOK4}
+ )
+ hook $HOOK
fi
# All done, rest is done by cleanup hook.