summaryrefslogtreecommitdiff
path: root/bin/ftpsync
blob: e8c30c473feaf12c1e8fb0c10a3cdbe9c16af3cc (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
#! /bin/bash
# No, we can not deal with sh alone.

set -e
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the function is
# the important part here)
set -E

# ftpsync script for Debian
# Based losely on a number of existing scripts, written by an
# unknown number of different people over the years.
#
# Copyright (C) 2008,2009 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# In case the admin somehow wants to have this script located someplace else,
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
# How the admin sets this isn't our place to deal with. One could use a wrapper
# for that. Or pam_env. Or whatever fits in the local setup. :)
BASEDIR=${BASEDIR:-"${HOME}"}

# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
# by Joerg Jaspert and the Debian mirroradm group.
# This is used to track which mirror is using which script version.
VERSION="80286"

# Source our common functions
. "${BASEDIR}/etc/common"

########################################################################
########################################################################
## functions                                                          ##
########################################################################
########################################################################
# We want to be able to get told what kind of sync we should do. This
# might be anything, from the archive to sync, the stage to do, etc. A
# list of currently understood and valid options is below. Multiple
# options are seperated by space. All the words have to have the word
# sync: in front or nothing will get used!
#
# Option        Behaviour
# stage1		 Only do stage1 sync
# stage2		 Only do stage2 sync
# all			 Do a complete sync
# archive:foo    Sync archive foo (if config for foo is available)
# callback       Call back when done (needs proper ssh setup for this to
#                work). It will always use the "command" callback:$HOSTNAME
#                where $HOSTNAME is the one defined below/in config and
#                will happen before slave mirrors are triggered.
#
# So to get us to sync all of the archive behind bpo and call back when
# we are done, a trigger command of
# "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the
# trick.
check_commandline() {
	while [ $# -gt 0 ]; do
		case "$1" in
			sync:stage1)
				SYNCSTAGE1="true"
				SYNCALL="false"
				;;
			sync:stage2)
				SYNCSTAGE2="true"
				SYNCALL="false"
				;;
			sync:callback)
				SYNCCALLBACK="true"
				;;
			sync:archive:*)
				ARCHIVE=${1##sync:archive:}
				# We do not like / or . in the remotely supplied archive name.
				ARCHIVE=${ARCHIVE//\/}
				ARCHIVE=${ARCHIVE//.}
				;;
			sync:all)
				SYNCALL="true"
				;;
			*)
				echo "Unknown option ${1} ignored"
				;;
		esac
		shift  # Check next set of parameters.
	done
}

# All the stuff we want to do when we exit, no matter where
cleanup() {
	trap - ERR TERM HUP INT QUIT EXIT
	# all done. Mail the log, exit.
	log "Mirrorsync done";
	if [ -n "${MAILTO}" ]; then
        # In case rsync had something on stderr
		if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
			mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
		fi
		if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
			# And the normal log
			MAILFILES="${LOG}"
			if [ "x${FULLLOGS}x" = "xtruex" ]; then
				# Someone wants full logs including rsync
				MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
			fi
			cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
		fi
	fi

	savelog "${LOGDIR}/rsync-${NAME}.log"
	savelog "${LOGDIR}/rsync-${NAME}.error"
	savelog "$LOG" > /dev/null

	rm -f "${LOCK}"
}

# Check rsyncs return value
check_rsync() {

	ret=$1
	msg=$2

	# 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
	# and us re-running. If it's not, uplink is broken anyways.
	case "${ret}" in
		0) return 0;;
		24) return 0;;
		23) return 2;;
		30) return 2;;
		*)
			error "ERROR: ${msg}"
			return 1
			;;
	esac
}

########################################################################
########################################################################


# As what are we called?
NAME="`basename $0`"
# The original command line arguments need to be saved!
if [ $# -gt 0 ]; then
    ORIGINAL_COMMAND=$*
else
    ORIGINAL_COMMAND=""
fi

SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""}
# Now, check if we got told about stuff via ssh
if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then
    # We deliberately add "nothing" and ignore it right again, to avoid
	# people from outside putting some set options in the first place,
	# making us parse them...
	set "nothing" "${SSH_ORIGINAL_COMMAND}"
	shift
	# Yes, unqouted $* here. Or the function will only see it as one
	# parameter, which doesnt help the case in it.
	check_commandline $*
fi

# Now, we can locally override all the above variables by just putting
# them into the .ssh/authorized_keys file forced command.
if [ -n "${ORIGINAL_COMMAND}" ]; then
	set ${ORIGINAL_COMMAND}
	check_commandline $*
fi

# If we have been told to do stuff for a different archive than default,
# set the name accordingly.
ARCHIVE=${ARCHIVE:-""}
if [ -n "${ARCHIVE}" ]; then
	NAME="${NAME}-${ARCHIVE}"
fi

# Now source the config for the archive we run on.
# (Yes, people can also overwrite the options above in the config file
# if they want to)
if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then
	. "${BASEDIR}/etc/${NAME}.conf"
else
	echo "Nono, you can't tell us about random archives. Bad boy!"
	exit 1
fi

########################################################################
# Config options go here. Feel free to overwrite them in the config    #
# file if you need to.                                                 #
# On debian.org machines the defaults should be ok.                    #
#                                                                      #
# The following extra variables can be defined in the config file:     #
#                                                                      #
# ARCH_EXCLUDE                                                         #
#  can be used to exclude a complete architecture from                 #
# mirrorring. Use as space seperated list.                             #
# Possible values are:                                                 #
# alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64,               #
# m68k, mipsel, mips, powerpc, s390, sh, sparc and source              #
# eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc"            #
#                                                                      #
# An unset value will mirror all architectures                         #
########################################################################

########################################################################
# There should be nothing to edit here, use the config file            #
########################################################################
MIRRORNAME=${MIRRORNAME:-`hostname -f`}
# Where to put logfiles in
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
# Our own logfile
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}

# Where should we put all the mirrored files?
TO=${TO:-"/org/ftp.debian.org/ftp/"}

# used by log() and error()
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}

# Where to send mails about mirroring to?
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
	# We are not on a debian.org host
	MAILTO=${MAILTO:-"root"}
else
	# Yay, on a .debian.org host
	MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
# Want errors only or every log?
ERRORSONLY=${ERRORSONLY:-"true"}
# Want full logs, ie. including the rsync one?
FULLLOGS=${FULLLOGS:-"false"}

# How many logfiles to keep
LOGROTATE=${LOGROTATE:-14}

# Our lockfile
LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"}
# Do we need another rsync run?
UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
# Trace file for mirror stats and checks (make sure we get full hostname)
TRACE=${TRACE:-"project/trace/${MIRRORNAME}"}

# rsync program
RSYNC=${RSYNC:-rsync}
# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
# excluded files
RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"}
# Default rsync options for *every* rsync call
RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-rltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
# Options we only use in the first pass, where we do not want packages/sources to fly in yet and dont want to delete files
RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude ls-lR*"}
# Options for the second pass, where we do want everything, including deletion of old and now unused files
RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
# Which rsync share to use on our upstream mirror?
RSYNC_PATH=${RSYNC_PATH:-"ftp"}

# our username for the rsync share
RSYNC_USER=${RSYNC_USER:-""}
# the password
RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}

# a possible proxy
RSYNC_PROXY=${RSYNC_PROXY:-""}

# Do we sync stage1?
SYNCSTAGE1=${SYNCSTAGE1:-"false"}
# Do we sync stage2?
SYNCSTAGE2=${SYNCSTAGE2:-"false"}
# Do we sync all?
SYNCALL=${SYNCALL:-"true"}
# Do we callback?
SYNCCALLBACK=${SYNCCALLBACK:-"false"}
# If we call back we need some more options defined in the config file.
CALLBACKUSER=${CALLBACKUSER:-"archvsync"}
CALLBACKHOST=${CALLBACKHOST:-"none"}
CALLBACKKEY=${CALLBACKKEY:-"none"}

# General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that!
EXCLUDE=${EXCLUDE:-""}

# The temp directory used by rsync --delay-updates is not
# world-readable remotely. Always exclude it to avoid errors. 
EXCLUDE="${EXCLUDE} --exclude .~tmp~/"

SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""}
ARCH_EXCLUDE=${ARCH_EXCLUDE:-""}
# Exclude architectures defined in $ARCH_EXCLUDE
for ARCH in ${ARCH_EXCLUDE}; do
    EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude disks-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes"
    if [ "${ARCH}" = "source" ]; then
		if [ -z ${SOURCE_EXCLUDE} ]; then
			SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc "
		fi
	fi
done

# Hooks
HOOK1=${HOOK1:-""}
HOOK2=${HOOK2:-""}
HOOK3=${HOOK3:-""}
HOOK4=${HOOK4:-""}
HOOK5=${HOOK5:-""}

# Are we a hub?
HUB=${HUB:-"false"}

# Some sane defaults
cd "${BASEDIR}"
umask 022

# If we are here for the first time, create the
# destination and the trace directory
mkdir -p "${TO}/project/trace"

# Used to make sure we will have the archive fully and completly synced before
# we stop, even if we get multiple pushes while this script is running.
# Otherwise we can end up with a half-synced archive:
# - get a push
# - sync, while locked
# - get another push. Of course no extra sync run then happens, we are locked.
# - done. Archive not correctly synced, we don't have all the changes from the second push.
touch "${UPDATEREQUIRED}"

# Check to see if another sync is in progress
if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
	if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
		# Process does either not exist or is not owned by us.
		echo "$$" > "${LOCK}"
	else
		echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
		exit 1
	fi
fi

trap cleanup EXIT ERR TERM HUP INT QUIT

# Start log by redirecting everything there.
exec >"$LOG" 2>&1 </dev/null

# Look who pushed us and note that in the log.
log "Mirrorsync start"
PUSHFROM="${SSH_CONNECTION%%\ *}"
if [ -n "${PUSHFROM}" ]; then
	log "We got pushed from ${PUSHFROM}"
fi
log "Acquired main lock"

if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
	if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then
		SYNCCALLBACK="false"
		error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
	fi
fi

HOOK=(
	HOOKNR=1
	HOOKSCR=${HOOK1}
)
hook $HOOK

# Now, we might want to sync from anonymous too.
# This is that deep in this script so hook1 could, if wanted, change things!
if [ -z ${RSYNC_USER} ]; then
	RSYNCPTH="${RSYNC_HOST}"
else
	RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
fi

# Now do the actual mirroring, and run as long as we have an updaterequired file.
export RSYNC_PASSWORD
export RSYNC_PROXY

while [ -e "${UPDATEREQUIRED}" ]; do
	log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"

	# if we want stage1 *or* all
	if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
		while [ -e "${UPDATEREQUIRED}" ]; do
			rm -f "${UPDATEREQUIRED}"
			log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE}  ${RSYNCPTH}::${RSYNC_PATH} ${TO}"

			set +e
			# Step one, sync everything except Packages/Releases
			${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
				${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
			result=$?
			set -e

			log "Back from rsync with returncode ${result}"
		done
	else
		# Fake a good resultcode
		result=0
	fi # Sync stage 1?
	rm -f "${UPDATEREQUIRED}"

	set +e
	check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
	GO=$?
	set -e
	if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
		log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
	elif [ ${GO} -ne 0 ]; then
		 exit 3
	fi

	HOOK=(
		HOOKNR=2
		HOOKSCR=${HOOK2}
	)
	hook $HOOK

	# if we want stage2 *or* all
	if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
		log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"

		set +e
		# We are lucky, it worked. Now do step 2 and sync again, this time including
		# the packages/releases files
		${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
			${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>${LOGDIR}/rsync-${NAME}.log 2>>${LOGDIR}/rsync-${NAME}.error
		result=$?
		set -e

		log "Back from rsync with returncode ${result}"
	else
		# Fake a good resultcode
		result=0
	fi # Sync stage 2?

	set +e
	check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
	GO=$?
	set -e
	if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
		log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
	elif [ ${GO} -ne 0 ]; then
		 exit 4
	fi

	HOOK=(
		HOOKNR=3
		HOOKSCR=${HOOK3}
	)
	hook $HOOK
done

if [ -d "`dirname "${TO}/${TRACE}"`" ]; then
      LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
      echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
	  echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
fi

HOOK=(
	HOOKNR=4
	HOOKSCR=${HOOK4}
)
hook $HOOK

if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
	set +e
	callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
	set -e
fi

if [ x${HUB} = "xtrue" ]; then
	# Only trigger slave mirrors if we had a push for stage2 or all
	if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
		log "Trigger slave mirrors"
		${BASEDIR}/bin/runmirrors "${ARCHIVE}"
		log "Trigger slave done"

		HOOK=(
			HOOKNR=5
			HOOKSCR=${HOOK5}
		)
		hook $HOOK
	fi
fi

# All done, rest is done by cleanup hook.