Comments

describe our varibales
Signed-off-by: Joerg Jaspert <joerg@debian.org>

3 files changed, 249 insertions, 5 deletions

diff --git a/bin/#ftpsync# b/bin/#ftpsync#
new file mode 100755
index 0000000..20281e9
--- /dev/null
+++ b/bin/#ftpsync#
@@ -0,0 +1,233 @@
+#! /bin/bash
+
+set -e
+
+# ftpsync script for Debian
+# Based losely on a number of existing scripts, written by an
+# unknown number of different people over the years.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# Source our common functions
+. ${HOME}/etc/common
+
+# Read our config file
+NAME="`basename $0`"
+
+# In case we are called with an argument we look for a different configuration.
+if [ -n $1 ];
+	NAME="${NAME}-$1"
+fi
+. ${HOME}/etc/${NAME}.conf
+
+########################################################################
+# Config options go here. Feel free to overwrite them in the config    #
+# file if you need to.                                                 #
+# On debian.org machines the defaults should be ok.                    #
+#                                                                      #
+# The following extra variables can be defined in the config file:     #
+#                                                                      #
+# ARCH_EXCLUDE                                                         #
+#  can be used to exclude a complete architecture from                 #
+# mirrorring. Use as space seperated list.                             #
+# Possible values are:                                                 #
+# alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64,               #
+# m68k, mipsel, mips, powerpc, s390, sh, sparc and source              #
+# eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc"            #
+#                                                                      #
+# An unset value will mirror all architectures                         #
+########################################################################
+
+########################################################################
+# There should be nothing to edit here, use the config file            #
+########################################################################
+HOSTNAME=${HOSTNAME:-`hostname -f`}
+# Where to put logfiles in
+LOGDIR=${LOGDIR:-"${HOME}/log"}
+# Our own logfile
+LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
+
+# Where should we put all the mirrored files?
+TO=${TO:-"/org/ftp.debian.org/ftp/"}
+
+# used by log()
+PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
+
+# Where to send mails about mirroring to?
+MAILTO=${MAILTO:-"joerg@debian.org"}
+
+# How to rotate our log
+SAVELOG=${SAVELOG:-"savelog -t -c 14"}
+# Lockfile program
+LOCKFILE=${LOCKFILE:-"lockfile"}
+
+# Our lockfile
+LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${HOSTNAME}"}
+# Do we need another rsync run?
+UPDATEREQUIRED="${TO}/Archive-Update-Required-${HOSTNAME}"
+# Trace file for mirror stats and checks
+TRACE=${TRACE:-"project/trace/${HOSTNAME}"}
+
+# rsync program
+RSYNC=${RSYNC:-rsync}
+# Default rsync options for *every* rsync call
+RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-rltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${HOSTNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${HOSTNAME}"}
+# Options we only use in the first pass, where we do not want packages/sources to fly in yet and dont want to delete files
+RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude ls-lR*"}
+# Options for the second pass, where we do want everything, including deletion of old and now unused files
+RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after"}
+# Which rsync share to use on our upstream mirror?
+RSYNC_PATH=${RSYNC_PATH:-"ftp"}
+
+# General excludes. Dont list architecture specific stuff here, use ARCH_EXCLUDE for that!
+EXCLUDE=${EXCLUDE:-""}
+
+# The temp directory used by rsync --delay-updates is not
+# world-readable remotely. Always exclude it to avoid errors. 
+EXCLUDE="${EXCLUDE} --exclude .~tmp~/"
+
+
+SOURCE_EXCLUDE=
+# Exclude architectures defined in $ARCH_EXCLUDE
+for ARCH in ${ARCH_EXCLUDE}; do
+    EXCLUDE=${EXCLUDE}"\
+                --exclude binary-${ARCH}/ \
+                --exclude disks-${ARCH}/ \
+                --exclude installer-${ARCH}/ \
+                --exclude Contents-${ARCH}.gz \
+                --exclude Contents-${ARCH}.diff/ \
+                --exclude arch-${ARCH}.files \
+                --exclude arch-${ARCH}.list.gz \
+                --exclude *_${ARCH}.deb \
+                --exclude *_${ARCH}.udeb "
+    if [ "${ARCH}" = "source" ]; then
+        SOURCE_EXCLUDE="\
+                --exclude source/ \
+                --exclude *.tar.gz \
+                --exclude *.diff.gz \
+                --exclude *.dsc "
+    fi
+done
+
+# Some sane defaults
+cd ${HOME}
+umask 002
+
+# Used to make sure we will have the archive fully and completly synced before
+# we stop, even if we get multiple pushes while this script is running.
+# Otherwise we can end up with a half-synced archive:
+# - get a push
+# - sync, while locked
+# - get another push. Of course no extra sync run then happens, we are locked.
+# - done. Archive not correctly synced, we don't have all the changes from the second push.
+touch "${UPDATEREQUIRED}"
+
+# If we are here for the first time, create the
+# destination and the trace directory
+mkdir -p ${TO}/project/trace
+
+# Check to see if another sync is in progress
+if ${LOCKFILE} -! -l 43200 -r 0 "${LOCK}"; then
+        error "Unable to start rsync, lock file still exists"
+        exit 1
+fi
+trap 'rm -f ${LOCK}; log "Mirrorsync done"; ${SAVELOG} "$LOG" > /dev/null' EXIT  
+
+# Start log by redirecting everything there.
+exec >"$LOG" 2>&1
+log "Mirrorsync start"
+log "Acquired main lock"
+
+export RSYNC_PASSWORD
+
+if [ -n ${HOOK1} ]; then
+	log "Running hook1: ${HOOK1}"
+	${HOOK1}
+	result=$?
+	log "Back from hook1, got returncode ${result}"
+fi
+
+# Now do the actual mirroring, and run as long as we have an updaterequired file.
+while [ -e "${UPDATEREQUIRED}" ]; do
+	log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
+	rm -f "${UPDATEREQUIRED}"
+
+	# Step one, sync everything except Packages/Releases
+	${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+		${RSYNC_USER}@${RSYNC_HOST}::${RSYNC_PATH} ${TO}
+
+	result=$?
+	# 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
+	# and us re-running. If it's not, uplink is broken anyways.
+	if [ $result -eq 0 ] || [ $result -eq 24 ]; then
+
+		if [ -n ${HOOK2} ]; then
+			log "Running hook2: ${HOOK1}"
+			${HOOK2}
+			result=$?
+			log "Back from hook1, got returncode ${result}"
+		fi
+
+		# We are lucky, it worked. Now do step 2 and sync again, this time including
+		# the packages/releases files
+		${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
+			${RSYNC_USER}@${RSYNC_HOST}::${RSYNC_PATH} ${TO}
+		result=$?
+		if [ $result -eq 0 ] || [ $result -eq 24 ]; then
+			if [ -n ${HOOK3} ]; then
+				log "Running hook3: ${HOOK1}"
+				${HOOK3}
+				result=$?
+				log "Back from hook3, got returncode ${result}"
+			fi
+		else
+			error "ERROR: Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
+			exit 4
+		fi
+	else
+		error "ERROR: Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
+		exit 3
+	fi
+done
+
+if [ -d "${TO}/project/trace" ]; then
+      date -u > "${TO}/project/trace/${HOSTNAME}"
+fi
+
+if [ -n ${HOOK4} ]; then
+	log "Running hook4: ${HOOK1}"
+	${HOOK4}
+	result=$?
+	log "Back from hook4, got returncode ${result}"
+fi
+
+if [ x${HUB} = "xtrue" ]; then
+	log "Trigger slave mirrors"
+	${HOME}/bin/runmirrors
+	log "Trigger slave done"
+
+	if [ -n ${HOOK5} ]; then
+		log "Running hook5: ${HOOK1}"
+		${HOOK5}
+		result=$?
+		log "Back from hook5, got returncode ${result}"
+	fi
+fi
+
+# Finally, all done. Mail the log, exit.
+if [ -n "${MAILTO}" ]; then
+	mail -e -s "${PROGRAM} ($(hostname)) - archive sync finished on $(date +"+%Y.%m.%d-%H:%M:%S")" ${MAILTO} < ${LOG}
+fi
+
+rm -f $LOCK
diff --git a/bin/.#ftpsync b/bin/.#ftpsync
new file mode 120000
index 0000000..e75045f
--- /dev/null
+++ b/bin/.#ftpsync
@@ -0,0 +1 @@
+joerg@vorlon.ganneff.de.13037:1222297872
\ No newline at end of file
diff --git a/bin/ftpsync b/bin/ftpsync
index 1d87b22..68c640d 100755
--- a/bin/ftpsync
+++ b/bin/ftpsync
@@ -53,31 +53,41 @@ fi
 # There should be nothing to edit here, use the config file            #
 ########################################################################
 HOSTNAME=${HOSTNAME:-`hostname -f`}
+# Where to put logfiles in
 LOGDIR=${LOGDIR:-"${HOME}/log"}
+# Our own logfile
 LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
-TO="/org/ftp.debian.org/ftp/"
+
+# Where should we put all the mirrored files?
+TO=${TO:-"/org/ftp.debian.org/ftp/"}
 
 # used by log()
-PROGRAM=${PROGRAM:-"ftpsync-$(hostname -s)"}
+PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
 
 # Where to send mails about mirroring to?
 MAILTO=${MAILTO:-"joerg@debian.org"}
 
+# How to rotate our log
 SAVELOG=${SAVELOG:-"savelog -t -c 14"}
+# Lockfile program
 LOCKFILE=${LOCKFILE:-"lockfile"}
 
+# Our lockfile
 LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${HOSTNAME}"}
+# Do we need another rsync run?
 UPDATEREQUIRED="${TO}/Archive-Update-Required-${HOSTNAME}"
+# Trace file for mirror stats and checks
 TRACE=${TRACE:-"project/trace/${HOSTNAME}"}
 
+# rsync program
 RSYNC=${RSYNC:-rsync}
 # Default rsync options for *every* rsync call
 RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-rltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${HOSTNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${HOSTNAME}"}
-# Options we only use in the first pass, where we do not want packages/sources to fly in yet
+# Options we only use in the first pass, where we do not want packages/sources to fly in yet and dont want to delete files
 RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude ls-lR*"}
-# Options for the second pass, where we do want everything
+# Options for the second pass, where we do want everything, including deletion of old and now unused files
 RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after"}
-# Which rsync share to use?
+# Which rsync share to use on our upstream mirror?
 RSYNC_PATH=${RSYNC_PATH:-"ftp"}
 
 # General excludes. Dont list architecture specific stuff here, use ARCH_EXCLUDE for that!