summaryrefslogtreecommitdiff
path: root/bin/ftpsync
diff options
context:
space:
mode:
authorJoerg Jaspert <joerg@debian.org>2010-01-02 22:10:26 +0100
committerJoerg Jaspert <joerg@debian.org>2010-01-02 22:10:26 +0100
commit5d9719e346af8580cb0e63227e83413fa2fa0f56 (patch)
tree3877f77eb5e4bd9c054a665af55e05238ca8efc8 /bin/ftpsync
parentea27d5078e60133428a445fb582d08ee8150e997 (diff)
space, lotsa space
change all the indentation stuff to match one schema. 4 spaces indent, no tabs. was pretty mixed up. Signed-off-by: Joerg Jaspert <joerg@debian.org>
Diffstat (limited to 'bin/ftpsync')
-rwxr-xr-xbin/ftpsync411
1 files changed, 205 insertions, 206 deletions
diff --git a/bin/ftpsync b/bin/ftpsync
index f70d506..6e9daeb 100755
--- a/bin/ftpsync
+++ b/bin/ftpsync
@@ -68,44 +68,44 @@ VERSION="80286"
# "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the
# trick.
check_commandline() {
- while [ $# -gt 0 ]; do
- case "$1" in
- sync:stage1)
- SYNCSTAGE1="true"
- SYNCALL="false"
- ;;
- sync:stage2)
- SYNCSTAGE2="true"
- SYNCALL="false"
- ;;
- sync:callback)
- SYNCCALLBACK="true"
- ;;
- sync:archive:*)
- ARCHIVE=${1##sync:archive:}
- # We do not like / or . in the remotely supplied archive name.
- ARCHIVE=${ARCHIVE//\/}
- ARCHIVE=${ARCHIVE//.}
- ;;
- sync:all)
- SYNCALL="true"
- ;;
- sync:mhop)
- SYNCMHOP="true"
- ;;
- *)
- echo "Unknown option ${1} ignored"
- ;;
- esac
- shift # Check next set of parameters.
- done
+ while [ $# -gt 0 ]; do
+ case "$1" in
+ sync:stage1)
+ SYNCSTAGE1="true"
+ SYNCALL="false"
+ ;;
+ sync:stage2)
+ SYNCSTAGE2="true"
+ SYNCALL="false"
+ ;;
+ sync:callback)
+ SYNCCALLBACK="true"
+ ;;
+ sync:archive:*)
+ ARCHIVE=${1##sync:archive:}
+ # We do not like / or . in the remotely supplied archive name.
+ ARCHIVE=${ARCHIVE//\/}
+ ARCHIVE=${ARCHIVE//.}
+ ;;
+ sync:all)
+ SYNCALL="true"
+ ;;
+ sync:mhop)
+ SYNCMHOP="true"
+ ;;
+ *)
+ echo "Unknown option ${1} ignored"
+ ;;
+ esac
+ shift # Check next set of parameters.
+ done
}
# All the stuff we want to do when we exit, no matter where
cleanup() {
- trap - ERR TERM HUP INT QUIT EXIT
- # all done. Mail the log, exit.
- log "Mirrorsync done";
+ trap - ERR TERM HUP INT QUIT EXIT
+ # all done. Mail the log, exit.
+ log "Mirrorsync done";
# Lets get a statistical value
SPEED=$(
@@ -121,47 +121,46 @@ cleanup() {
fi
log "Rsync transfer speed: ${SPEED} KB/s"
- if [ -n "${MAILTO}" ]; then
+ if [ -n "${MAILTO}" ]; then
# In case rsync had something on stderr
- if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
- mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
- fi
- if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
- # And the normal log
- MAILFILES="${LOG}"
- if [ "x${FULLLOGS}x" = "xtruex" ]; then
- # Someone wants full logs including rsync
- MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
- fi
- cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
- fi
- fi
-
- savelog "${LOGDIR}/rsync-${NAME}.log"
- savelog "${LOGDIR}/rsync-${NAME}.error"
- savelog "$LOG" > /dev/null
-
- rm -f "${LOCK}"
+ if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
+ mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
+ fi
+ if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
+ # And the normal log
+ MAILFILES="${LOG}"
+ if [ "x${FULLLOGS}x" = "xtruex" ]; then
+ # Someone wants full logs including rsync
+ MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
+ fi
+ cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
+ fi
+ fi
+
+ savelog "${LOGDIR}/rsync-${NAME}.log"
+ savelog "${LOGDIR}/rsync-${NAME}.error"
+ savelog "$LOG" > /dev/null
+
+ rm -f "${LOCK}"
}
# Check rsyncs return value
check_rsync() {
-
- ret=$1
- msg=$2
-
- # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
- # and us re-running. If it's not, uplink is broken anyways.
- case "${ret}" in
- 0) return 0;;
- 24) return 0;;
- 23) return 2;;
- 30) return 2;;
- *)
- error "ERROR: ${msg}"
- return 1
- ;;
- esac
+ ret=$1
+ msg=$2
+
+ # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
+ # and us re-running. If it's not, uplink is broken anyways.
+ case "${ret}" in
+ 0) return 0;;
+ 24) return 0;;
+ 23) return 2;;
+ 30) return 2;;
+ *)
+ error "ERROR: ${msg}"
+ return 1
+ ;;
+ esac
}
########################################################################
@@ -181,37 +180,37 @@ SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""}
# Now, check if we got told about stuff via ssh
if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then
# We deliberately add "nothing" and ignore it right again, to avoid
- # people from outside putting some set options in the first place,
- # making us parse them...
- set "nothing" "${SSH_ORIGINAL_COMMAND}"
- shift
- # Yes, unqouted $* here. Or the function will only see it as one
- # parameter, which doesnt help the case in it.
- check_commandline $*
+ # people from outside putting some set options in the first place,
+ # making us parse them...
+ set "nothing" "${SSH_ORIGINAL_COMMAND}"
+ shift
+ # Yes, unqouted $* here. Or the function will only see it as one
+ # parameter, which doesnt help the case in it.
+ check_commandline $*
fi
# Now, we can locally override all the above variables by just putting
# them into the .ssh/authorized_keys file forced command.
if [ -n "${ORIGINAL_COMMAND}" ]; then
- set ${ORIGINAL_COMMAND}
- check_commandline $*
+ set ${ORIGINAL_COMMAND}
+ check_commandline $*
fi
# If we have been told to do stuff for a different archive than default,
# set the name accordingly.
ARCHIVE=${ARCHIVE:-""}
if [ -n "${ARCHIVE}" ]; then
- NAME="${NAME}-${ARCHIVE}"
+ NAME="${NAME}-${ARCHIVE}"
fi
# Now source the config for the archive we run on.
# (Yes, people can also overwrite the options above in the config file
# if they want to)
if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then
- . "${BASEDIR}/etc/${NAME}.conf"
+ . "${BASEDIR}/etc/${NAME}.conf"
else
- echo "Nono, you can't tell us about random archives. Bad boy!"
- exit 1
+ echo "Nono, you can't tell us about random archives. Bad boy!"
+ exit 1
fi
########################################################################
@@ -250,11 +249,11 @@ PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
# Where to send mails about mirroring to?
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
- # We are not on a debian.org host
- MAILTO=${MAILTO:-"root"}
+ # We are not on a debian.org host
+ MAILTO=${MAILTO:-"root"}
else
- # Yay, on a .debian.org host
- MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
+ # Yay, on a .debian.org host
+ MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
# Want errors only or every log?
ERRORSONLY=${ERRORSONLY:-"true"}
@@ -330,10 +329,10 @@ ARCH_EXCLUDE=${ARCH_EXCLUDE:-""}
for ARCH in ${ARCH_EXCLUDE}; do
EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude disks-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes"
if [ "${ARCH}" = "source" ]; then
- if [ -z ${SOURCE_EXCLUDE} ]; then
- SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc "
- fi
- fi
+ if [ -z ${SOURCE_EXCLUDE} ]; then
+ SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc "
+ fi
+ fi
done
# Hooks
@@ -370,29 +369,29 @@ function mainroutine() {
log "Mirrorsync start"
PUSHFROM="${SSH_CONNECTION%%\ *}"
if [ -n "${PUSHFROM}" ]; then
- log "We got pushed from ${PUSHFROM}"
+ log "We got pushed from ${PUSHFROM}"
fi
log "Acquired main lock"
if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
- if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then
- SYNCCALLBACK="false"
- error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
- fi
+ if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then
+ SYNCCALLBACK="false"
+ error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
+ fi
fi
HOOK=(
- HOOKNR=1
- HOOKSCR=${HOOK1}
+ HOOKNR=1
+ HOOKSCR=${HOOK1}
)
hook $HOOK
# Now, we might want to sync from anonymous too.
# This is that deep in this script so hook1 could, if wanted, change things!
if [ -z ${RSYNC_USER} ]; then
- RSYNCPTH="${RSYNC_HOST}"
+ RSYNCPTH="${RSYNC_HOST}"
else
- RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
+ RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
fi
# Now do the actual mirroring, and run as long as we have an updaterequired file.
@@ -400,137 +399,137 @@ function mainroutine() {
export RSYNC_PROXY
while [ -e "${UPDATEREQUIRED}" ]; do
- log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
+ log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
# if we want stage1 *or* all
- if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
- while [ -e "${UPDATEREQUIRED}" ]; do
- rm -f "${UPDATEREQUIRED}"
- log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
-
- set +e
- # Step one, sync everything except Packages/Releases
- ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
- ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
- result=$?
- set -e
-
- log "Back from rsync with returncode ${result}"
- done
- else
- # Fake a good resultcode
- result=0
- fi # Sync stage 1?
- rm -f "${UPDATEREQUIRED}"
-
- set +e
- check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
- GO=$?
- set -e
- if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
- log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
- elif [ ${GO} -ne 0 ]; then
- exit 3
- fi
-
- HOOK=(
- HOOKNR=2
- HOOKSCR=${HOOK2}
- )
- hook $HOOK
-
- # if we want stage2 *or* all
- if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
- log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
-
- set +e
- # We are lucky, it worked. Now do step 2 and sync again, this time including
- # the packages/releases files
- ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
- ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error
- result=$?
- set -e
-
- log "Back from rsync with returncode ${result}"
- else
- # Fake a good resultcode
- result=0
- fi # Sync stage 2?
-
- set +e
- check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
- GO=$?
- set -e
- if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
- log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
- elif [ ${GO} -ne 0 ]; then
- exit 4
- fi
-
- HOOK=(
- HOOKNR=3
- HOOKSCR=${HOOK3}
- )
- hook $HOOK
+ if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+ while [ -e "${UPDATEREQUIRED}" ]; do
+ rm -f "${UPDATEREQUIRED}"
+ log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+
+ set +e
+ # Step one, sync everything except Packages/Releases
+ ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+ ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
+ result=$?
+ set -e
+
+ log "Back from rsync with returncode ${result}"
+ done
+ else
+ # Fake a good resultcode
+ result=0
+ fi # Sync stage 1?
+ rm -f "${UPDATEREQUIRED}"
+
+ set +e
+ check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
+ GO=$?
+ set -e
+ if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
+ log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
+ elif [ ${GO} -ne 0 ]; then
+ exit 3
+ fi
+
+ HOOK=(
+ HOOKNR=2
+ HOOKSCR=${HOOK2}
+ )
+ hook $HOOK
+
+ # if we want stage2 *or* all
+ if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+ log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
+
+ set +e
+ # We are lucky, it worked. Now do step 2 and sync again, this time including
+ # the packages/releases files
+ ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+ ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error
+ result=$?
+ set -e
+
+ log "Back from rsync with returncode ${result}"
+ else
+ # Fake a good resultcode
+ result=0
+ fi # Sync stage 2?
+
+ set +e
+ check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
+ GO=$?
+ set -e
+ if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
+ log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
+ elif [ ${GO} -ne 0 ]; then
+ exit 4
+ fi
+
+ HOOK=(
+ HOOKNR=3
+ HOOKSCR=${HOOK3}
+ )
+ hook $HOOK
done
# We only update our tracefile when we had a stage2 or an all sync.
# Otherwise we would update it after stage1 already, which is wrong.
- if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
+ if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
if [ -d "`dirname "${TO}/${TRACE}"`" ]; then
LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
- echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
+ echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
fi
fi
HOOK=(
- HOOKNR=4
- HOOKSCR=${HOOK4}
+ HOOKNR=4
+ HOOKSCR=${HOOK4}
)
hook $HOOK
if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
- set +e
- callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
- set -e
+ set +e
+ callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
+ set -e
fi
# Remove the Archive-Update-in-Progress file before we push our downstreams.
- rm -f "${LOCK}"
+ rm -f "${LOCK}"
if [ x${HUB} = "xtrue" ]; then
- # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
- if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then
+ # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
+ if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then
RUNMIRRORARGS=""
if [ -n "${ARCHIVE}" ]; then
# We tell runmirrors about the archive we are running on.
- RUNMIRRORARGS="-a ${ARCHIVE}"
+ RUNMIRRORARGS="-a ${ARCHIVE}"
fi
- # We also tell runmirrors that we are running it from within ftpsync, so it can change
- # the way it works with mhop based on that.
- RUNMIRRORARGS="${RUNMIRRORARGS} -f"
-
- if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then
- # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
- # not get to this point.
- # So if that happens, tell runmirrors we are doing mhop
- RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
- elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then
- RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
- elif [ "xtruex" = "x${SYNCALL}x" ]; then
- RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
- fi
- log "Trigger slave mirrors using ${RUNMIRRORARGS}"
- ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS}
- log "Trigger slave done"
-
- HOOK=(
- HOOKNR=5
- HOOKSCR=${HOOK5}
- )
- hook $HOOK
- fi
+ # We also tell runmirrors that we are running it from within ftpsync, so it can change
+ # the way it works with mhop based on that.
+ RUNMIRRORARGS="${RUNMIRRORARGS} -f"
+
+ if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then
+ # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
+ # not get to this point.
+ # So if that happens, tell runmirrors we are doing mhop
+ RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
+ elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then
+ RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
+ elif [ "xtruex" = "x${SYNCALL}x" ]; then
+ RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
+ fi
+ log "Trigger slave mirrors using ${RUNMIRRORARGS}"
+ ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS}
+ log "Trigger slave done"
+
+ HOOK=(
+ HOOKNR=5
+ HOOKSCR=${HOOK5}
+ )
+ hook $HOOK
+ fi
fi
# All done, lets call cleanup
@@ -575,7 +574,7 @@ if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
unixtime=$(date +%s)
difference=$(( $unixtime - $stamptime ))
if [ ${difference} -ge ${LOCKTIMEOUT} ]; then
- # Took longer than 60 minutes? Assume it broke and take the lock
+ # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock
echo "$$" > "${LOCK}"
else
echo "Unable to start rsync, lock file younger than one hour"