1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
|
#! /bin/bash
set -e
# ftpsync script for Debian
# Based losely on a number of existing scripts, written by an
# unknown number of different people over the years.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# Source our common functions
. ${HOME}/etc/common
# Read our config file
NAME="`basename $0`"
# In case we are called with an argument we look for a different configuration.
if [ -n $1 ];
NAME="${NAME}-$1"
fi
. ${HOME}/etc/${NAME}.conf
########################################################################
# Config options go here. Feel free to overwrite them in the config #
# file if you need to. #
# On debian.org machines the defaults should be ok. #
# #
# The following extra variables can be defined in the config file: #
# #
# ARCH_EXCLUDE #
# can be used to exclude a complete architecture from #
# mirrorring. Use as space seperated list. #
# Possible values are: #
# alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, #
# m68k, mipsel, mips, powerpc, s390, sh, sparc and source #
# eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc" #
# #
# An unset value will mirror all architectures #
########################################################################
########################################################################
# There should be nothing to edit here, use the config file #
########################################################################
HOSTNAME=${HOSTNAME:-`hostname -f`}
# Where to put logfiles in
LOGDIR=${LOGDIR:-"${HOME}/log"}
# Our own logfile
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
# Where should we put all the mirrored files?
TO=${TO:-"/org/ftp.debian.org/ftp/"}
# used by log()
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
# Where to send mails about mirroring to?
MAILTO=${MAILTO:-"joerg@debian.org"}
# How to rotate our log
SAVELOG=${SAVELOG:-"savelog -t -c 14"}
# Lockfile program
LOCKFILE=${LOCKFILE:-"lockfile"}
# Our lockfile
LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${HOSTNAME}"}
# Do we need another rsync run?
UPDATEREQUIRED="${TO}/Archive-Update-Required-${HOSTNAME}"
# Trace file for mirror stats and checks
TRACE=${TRACE:-"project/trace/${HOSTNAME}"}
# rsync program
RSYNC=${RSYNC:-rsync}
# Default rsync options for *every* rsync call
RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-rltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${HOSTNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${HOSTNAME}"}
# Options we only use in the first pass, where we do not want packages/sources to fly in yet and dont want to delete files
RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude ls-lR*"}
# Options for the second pass, where we do want everything, including deletion of old and now unused files
RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after"}
# Which rsync share to use on our upstream mirror?
RSYNC_PATH=${RSYNC_PATH:-"ftp"}
# General excludes. Dont list architecture specific stuff here, use ARCH_EXCLUDE for that!
EXCLUDE=${EXCLUDE:-""}
# The temp directory used by rsync --delay-updates is not
# world-readable remotely. Always exclude it to avoid errors.
EXCLUDE="${EXCLUDE} --exclude .~tmp~/"
# Check for some environment variables
if [ -z "${TO}" ] || [ -z "${RSYNC_HOST}" ] || [ -z "${RSYNC_PATH}" ]; then
error "One of the TO, RSYNC_HOST or RSYNC_PATH variables seems to be empty"
exit 2
fi
SOURCE_EXCLUDE=
# Exclude architectures defined in $ARCH_EXCLUDE
for ARCH in ${ARCH_EXCLUDE}; do
EXCLUDE=${EXCLUDE}"\
--exclude binary-${ARCH}/ \
--exclude disks-${ARCH}/ \
--exclude installer-${ARCH}/ \
--exclude Contents-${ARCH}.gz \
--exclude Contents-${ARCH}.diff/ \
--exclude arch-${ARCH}.files \
--exclude arch-${ARCH}.list.gz \
--exclude *_${ARCH}.deb \
--exclude *_${ARCH}.udeb "
if [ "${ARCH}" = "source" ]; then
SOURCE_EXCLUDE="\
--exclude source/ \
--exclude *.tar.gz \
--exclude *.diff.gz \
--exclude *.dsc "
fi
done
# Some sane defaults
cd ${HOME}
umask 002
# Used to make sure we will have the archive fully and completly synced before
# we stop, even if we get multiple pushes while this script is running.
# Otherwise we can end up with a half-synced archive:
# - get a push
# - sync, while locked
# - get another push. Of course no extra sync run then happens.
# - done. Archive not correctly synced, we don't have all the changes from the second push.
touch "${UPDATEREQUIRED}"
# If we are here for the first time, create the
# destination and the trace directory
mkdir -p ${TO}/project/trace
# Check to see if another sync is in progress
if ${LOCKFILE} -! -l 43200 -r 0 "${LOCK}"; then
error "Unable to start rsync, lock file still exists"
exit 1
fi
trap 'rm -f ${LOCK}; log "Mirrorsync done"; ${SAVELOG} "$LOG" > /dev/null' EXIT
# Start log by redirecting everything there.
exec >"$LOG" 2>&1
log "Mirrorsync start"
log "Acquired main lock"
if [ -n ${HOOK1} ]; then
log "Running hook1: ${HOOK1}"
${HOOK1}
result=$?
log "Back from hook1, got returncode ${result}"
fi
export RSYNC_PASSWORD
# Now do the actual mirroring, and run as long as we have an updaterequired file.
while [ -e "${UPDATEREQUIRED}" ]; do
log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
rm -f "${UPDATEREQUIRED}"
# Step one, sync everything except Packages/Releases
${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
${RSYNC_USER}@${RSYNC_HOST}::${RSYNC_PATH} ${TO}
result=$?
# 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
# and us re-running. If it's not, uplink is broken anyways.
if [ $result -eq 0 ] || [ $result -eq 24 ]; then
if [ -n ${HOOK2} ]; then
log "Running hook2: ${HOOK1}"
${HOOK2}
result=$?
log "Back from hook1, got returncode ${result}"
fi
# We are lucky, it worked. Now do step 2 and sync again, this time including
# the packages/releases files
${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
${RSYNC_USER}@${RSYNC_HOST}::${RSYNC_PATH} ${TO}
result=$?
if [ $result -eq 0 ] || [ $result -eq 24 ]; then
if [ -n ${HOOK3} ]; then
log "Running hook3: ${HOOK1}"
${HOOK3}
result=$?
log "Back from hook3, got returncode ${result}"
fi
else
error "ERROR: Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
exit 4
fi
else
error "ERROR: Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
exit 3
fi
done
if [ -d "${TO}/project/trace" ]; then
date -u > "${TO}/project/trace/${HOSTNAME}"
fi
if [ -n ${HOOK4} ]; then
log "Running hook4: ${HOOK1}"
${HOOK4}
result=$?
log "Back from hook4, got returncode ${result}"
fi
if [ x${HUB} = "xtrue" ]; then
log "Trigger slave mirrors"
${HOME}/bin/runmirrors
log "Trigger slave done"
if [ -n ${HOOK5} ]; then
log "Running hook5: ${HOOK1}"
${HOOK5}
result=$?
log "Back from hook5, got returncode ${result}"
fi
fi
# Finally, all done. Mail the log, exit.
if [ -n "${MAILTO}" ]; then
mail -e -s "${PROGRAM} ($(hostname)) - archive sync finished on $(date +"+%Y.%m.%d-%H:%M:%S")" ${MAILTO} < ${LOG}
fi
rm -f $LOCK
|
