diff options
| -rwxr-xr-x | bin/ftpsync | 139 | ||||
| -rw-r--r-- | etc/common | 11 | ||||
| -rw-r--r-- | etc/ftpsync.conf.sample | 36 |
3 files changed, 163 insertions, 23 deletions
diff --git a/bin/ftpsync b/bin/ftpsync index 951aed7..2c41702 100755 --- a/bin/ftpsync +++ b/bin/ftpsync @@ -12,7 +12,7 @@ set -E # Based losely on a number of existing scripts, written by an # unknown number of different people over the years. # -# Copyright (C) 2008,2009,2010,2011 Joerg Jaspert <joerg@debian.org> +# Copyright (C) 2008-2012 Joerg Jaspert <joerg@debian.org> # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -36,7 +36,7 @@ BASEDIR=${BASEDIR:-"${HOME}"} # Script version. DO NOT CHANGE, *unless* you change the master copy maintained # by Joerg Jaspert and the Debian mirroradm group. # This is used to track which mirror is using which script version. -VERSION="80387" +VERSION="20120521" # Source our common functions . "${BASEDIR}/etc/common" @@ -276,6 +276,16 @@ UIPRETRIES=${UIPRETRIES:-3} UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}" # Trace file for mirror stats and checks (make sure we get full hostname) TRACE=${TRACE:-"project/trace/${MIRRORNAME}"} +# The trace file can have different format/contents. Here you can select +# what it will be. +# Possible values are +# "full" - all information +# "terse" - basic, timestamp only (date -u) +# "touch" - just touch the file in existance +# "none" - no tracefile at all +# +# Default and required value for Debian mirrors is full. +EXTENDEDTRACE=${EXTENDEDTRACE:-"full"} # rsync program RSYNC=${RSYNC:-rsync} @@ -284,18 +294,38 @@ RSYNC=${RSYNC:-rsync} RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"} # limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited RSYNC_BW=${RSYNC_BW:-0} +RSYNC_PROTOCOL=$(rsync_protocol) + +# Set the delete method to --delete-delay if protocol version is 30 or +# greater (meaning rsync 3.0.0 or greater is used). Use --delete-after +# otherwise. +if [ 30 -le $RSYNC_PROTOCOL ]; then + RSYNC_DELETE_METHOD=delay +else + RSYNC_DELETE_METHOD=after +fi + # Default rsync options for *every* rsync call RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"} # Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files -RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"} +RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --exclude=i18n/* --exclude=ls-lR*"} # Options for the second pass, where we do want everything, including deletion of old and now unused files -RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"} +RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-excluded"} # Which rsync share to use on our upstream mirror? RSYNC_PATH=${RSYNC_PATH:-"debian"} +# Extra rsync options as defined by the admin locally. Won't be set +# to any default by ftpsync. Those will be added to EACH AND EVERY rsync call. +RSYNC_EXTRA=${RSYNC_EXTRA:-""} + # Now add the bwlimit option. As default is 0 we always add it, rsync interprets # 0 as unlimited, so this is safe. -RSYNC_OPTIONS="--bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}" +RSYNC_OPTIONS="${RSYNC_EXTRA} --bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}" + +# Finally, make sure RSYNC_OPTIONS2 has either --delete-after or --deleter-delay +if ! [[ ${RSYNC_OPTIONS2} =~ --delete-(after|delay) ]]; then + RSYNC_OPTIONS2+=" --delete-${RSYNC_DELETE_METHOD}" +fi # We have no default host to sync from, but will error out if its unset RSYNC_HOST=${RSYNC_HOST:-""} @@ -320,7 +350,7 @@ SYNCSTAGE2=${SYNCSTAGE2:-"false"} SYNCALL=${SYNCALL:-"true"} # Do we have a mhop sync? SYNCMHOP=${SYNCMHOP:-"false"} -# Do we callback? +# Do we callback? (May get changed later) SYNCCALLBACK=${SYNCCALLBACK:-"false"} # If we call back we need some more options defined in the config file. CALLBACKUSER=${CALLBACKUSER:-"archvsync"} @@ -331,17 +361,17 @@ CALLBACKKEY=${CALLBACKKEY:-"none"} EXCLUDE=${EXCLUDE:-""} # The temp directory used by rsync --delay-updates is not -# world-readable remotely. Always exclude it to avoid errors. -EXCLUDE="${EXCLUDE} --exclude .~tmp~/" +# world-readable remotely. Always exclude it to avoid errors. +EXCLUDE="${EXCLUDE} --exclude=.~tmp~/" SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""} ARCH_EXCLUDE=${ARCH_EXCLUDE:-""} # Exclude architectures defined in $ARCH_EXCLUDE for ARCH in ${ARCH_EXCLUDE}; do - EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes" + EXCLUDE="${EXCLUDE} --exclude=binary-${ARCH}/ --exclude=installer-${ARCH}/ --exclude=Contents-${ARCH}.gz --exclude=Contents-udeb-${ARCH}.gz --exclude=Contents-${ARCH}.diff/ --exclude=arch-${ARCH}.files --exclude=arch-${ARCH}.list.gz --exclude=*_${ARCH}.deb --exclude=*_${ARCH}.udeb --exclude=*_${ARCH}.changes" if [ "${ARCH}" = "source" ]; then if [ -z ${SOURCE_EXCLUDE} ]; then - SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.tar.xz -exclude *.diff.bz2 --exclude *.dsc " + SOURCE_EXCLUDE=" --exclude=source/ --exclude=*.tar.gz --exclude=*.diff.gz --exclude=*.tar.bz2 --exclude=*.tar.xz --exclude=*.diff.bz2 --exclude=*.dsc " fi fi done @@ -383,11 +413,11 @@ if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then if [ ${BASH_VERSINFO[0]} -gt 3 ] || [ -L /proc/self ]; then # We have a recent enough bash version, lets do it the easy way, # the lock will contain the right pid, thanks to $BASHPID - if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then + if ! $(kill -0 $(< ${LOCK}) 2>/dev/null); then # Process does either not exist or is not owned by us. echo "$$" > "${LOCK}" else - echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})" + echo "Unable to start rsync, lock file still exists, PID $(< ${LOCK})" exit 1 fi else @@ -565,14 +595,46 @@ done # We only update our tracefile when we had a stage2 or an all sync. # Otherwise we would update it after stage1 already, which is wrong. + if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then - if [ -d "$(dirname "${TO}/${TRACE}")" ]; then - LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}" - echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}" - echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}" - fi + case ${EXTENDEDTRACE} in + none) + log "No trace file wanted. Not creating one" + ;; + touch) + log "Just touching the trace file" + touch "${TO}/${TRACE}" + ;; + terse|full) + log "Creating a ${EXTENDEDTRACE} trace file" + if [ -d "$(dirname "${TO}/${TRACE}")" ]; then + LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}.new" + echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}.new" + echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}.new" + if [ "xfullx" = "x${EXTENDEDTRACE}x" ]; then + GLOBALARCHLIST="source amd64 armel armhf hurd-i386 i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc" + + AEXCLUDE="^${ARCH_EXCLUDE// /\$|^}$" + ARCHLIST="" + for ARCH in ${GLOBALARCHLIST}; do + if ! [[ ${ARCH} =~ ${AEXCLUDE} ]]; then + ARCHLIST="${ARCHLIST} ${ARCH}" + fi + done + out="GUESSED:{${ARCHLIST}}" + echo "Architectures: ${out}" >> "${TO}/${TRACE}.new" + echo "Upstream-mirror: ${RSYNC_HOST}" >> "${TO}/${TRACE}.new" + fi # full trace + mv "${TO}/${TRACE}.new" "${TO}/${TRACE}" + fi + ;; + *) + error "Unsupported EXTENDEDTRACE value configured in ${BASEDIR}/etc/${NAME}.conf, please fix" + ;; + esac fi + HOOK=( HOOKNR=4 HOOKSCR=${HOOK4} @@ -588,6 +650,49 @@ fi # Remove the Archive-Update-in-Progress file before we push our downstreams. rm -f "${LOCK}" +# Check if there is a newer version of ftpsync. If so inform the admin, but not +# more than once every third day. +if [ -r "${TO}/project/ftpsync/LATEST.VERSION" ]; then + LATEST=$(< "${TO}/project/ftpsync/LATEST.VERSION") + if ! [[ ${LATEST} =~ [0-9]+ ]]; then + LATEST=0 + fi + if [ ${LATEST} -gt ${VERSION} ]; then + if [ -n "${MAILTO}" ]; then + difference=0 + if [ -f "${LOGDIR}/ftpsync.newversion" ]; then + stamptime=$(< "${LOGDIR}/ftpsync.newversion") + unixtime=$(date +%s) + difference=$(( $unixtime - $stamptime )) + fi + if [ ${difference} -ge 259200 ]; then + # Only warn every third day + mail -e -s "[$(hostname -s)] Update for ftpsync available" ${MAILTO} <<EOF +Hello admin, + +i found that there is a new version of me available. +Me lonely ftpsync is currently version: ${VERSION} +New release of myself is available as: ${LATEST} + +Me, myself and I - and the Debian mirroradmins - would be very grateful +if you could update me. You can find the latest version on your mirror, +check $(hostname -s):${TO}/project/ftpsync/ftpsync-${LATEST}.tar.gz + +You can ensure the validity of that file by using sha512sum or md5sum +against the available checksum files secured with a signature from the +Debian FTPMaster signing key. + +EOF + + date +%s > "${LOGDIR}/ftpsync.newversion" + fi + fi + else + # Remove a possible stampfile + rm -f "${LOGDIR}/ftpsync.newversion" + fi +fi + if [ x${HUB} = "xtrue" ]; then # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then @@ -181,7 +181,7 @@ log () { # to the address configured in MAILTO (if non-empty) error () { log "$@" - if [ -z "${MAILTO}" ]; then + if [ -n "${MAILTO}" ]; then echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO} fi } @@ -232,3 +232,12 @@ savelog() { done mv "${torotate}" "${torotate}.0" } + +# Return rsync version +rsync_protocol() { + RSYNC_VERSION="$(${RSYNC} --version)" + if [[ $RSYNC_VERSION =~ (protocol[ ]+version[ ]+([0-9]+)) ]]; then + echo ${BASH_REMATCH[2]} + fi + unset RSYNC_VERSION +} diff --git a/etc/ftpsync.conf.sample b/etc/ftpsync.conf.sample index e3f5cf5..c8750fe 100644 --- a/etc/ftpsync.conf.sample +++ b/etc/ftpsync.conf.sample @@ -104,30 +104,56 @@ ## This is *relative* to ${TO} #TRACE="project/trace/${MIRRORNAME}" +## The trace file can have different format/contents. Here you can select +## what it will be. +## Possible values are +## "full" - all information +## "terse" - basic, timestamp only (date -u) +## "touch" - just touch the file in existance +## "none" - no tracefile at all +## +## Default and required value for Debian mirrors is full. +#EXTENDEDTRACE="full" + ## We sync our mirror using rsync (everything else would be insane), so ## we need a few options set. ## The rsync program #RSYNC=rsync +## Extra rsync options as defined by the local admin. +## There is no default by ftpsync. +## +## Please note that these options are added to EVERY rsync call. +## Also note that these are added at the beginning of the rsync call, as +## the very first set of options. +## Please ensure you do not add a conflict with the usual rsync options as +## shown below. +# RSYNC_EXTRA="" + +## limit I/O bandwidth. Value is KBytes per second, unset or 0 means unlimited +#RSYNC_BW="" + ## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! ## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! ## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! ## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T! -## limit I/O bandwidth. Value is KBytes per second, unset or 0 means unlimited -#RSYNC_BW="" +## Files that must *never* be deleted by rsync. These are files handled +## internally by ftpsync and will be created, updated, and deleted when +## appropriate. +#RSYNC_FILTER="--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}" ## Default rsync options every rsync invocation sees. -#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}" +#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}" ## Options the first pass gets. We do not want the Packages/Source indices ## here, and we also do not want to delete any files yet. -#RSYNC_OPTIONS1="--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*" +#RSYNC_OPTIONS1="--exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --exclude=i18n/* --exclude=ls-lR*" ## Options the second pass gets. Now we want the Packages/Source indices too ## and we also want to delete files. We also want to delete files that are ## excluded. -#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded" +#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-excluded" ## You may establish the connection via a web proxy by setting the environment ## variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note |