summaryrefslogtreecommitdiff
path: root/bin/runmirrors
blob: 752e35167e6bdf777bfe8c944e019863e66cc01f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
#! /bin/bash

set -e
set -u

# runmirrors script for Debian
# Based losely on existing scripts, written by an unknown number of
# different people over the years.
#
# Copyright (C) 2008, 2009 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# In case the admin somehow wants to have this script located someplace else,
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
BASEDIR=${BASEDIR:-"${HOME}"}

NAME="`basename $0`"

HELP="$0, (C) 2008, 2009 by Joerg Jaspert <joerg@debian.org>\n
Usage:\n\n

1.) a single parameter with NO leading -.\n
\t  This will will then be used as the addition for our configfile. Ie. \`$0 security\` will\n
\t  have us look for ${NAME}-security.{conf,mirror} files.\n\n

2.) using getopt style parameters:\n
\t -a [NAME]   - Same as 1.) above, used for the config files. Default empty.\n
\t -k [TYPE]   - Type of push. all, stage2, mhop. Default mhop.\n
\t -f          - Run from within the mirrorscript ftpsync. Don't use from commandline!\n
\t -h          - Print this help and exit
"
# If we got options, lets see if we use newstyle options, or oldstyle. If oldstyle
# it will not start with a -. If we find oldstyle we assume its only one, the config
# name we run on.
if [ $# -gt 0 ]; then
    if [ "x${1:0:1}x" != "x-x" ]; then
        # Yes, does not start with a -, so use it for the config name.
        CONF=${1:-""}
        if [ -n "${CONF}" ]; then
            NAME="${NAME}-${CONF}"
        fi
    else
        # Yeah well, new style, starting with - for getopts
        while getopts ':a:k:fh' OPTION ; do
            case $OPTION in
                a)	CONF="${OPTARG}"
                    if [ -n "${CONF}" ]; then
                        NAME="${NAME}-${CONF}"
                    fi
                    ;;
                k)	PUSHKIND="${OPTARG}"
                    ;;
                f)	FROMFTPSYNC="true"
                    ;;
                h)	echo -e $HELP
                    exit 0
                    ;;

                *)  echo "Invalid usage"
		            echo -e $HELP 
                    exit 1
                    ;;
            esac
        done
    fi
fi
# Make sure the values are always defined, even if there was no commandline option
# for them
# Default config is empty
CONF=${CONF:-""}

# Set the default to all, if we didnt get told about it. Currently
# valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1,
# stage2 - staged push, second phase. Default is mhop.
PUSHKIND=${PUSHKIND:-"mhop"}

# If we are pushed from within ftpsync. Default false.
FROMFTPSYNC=${FROMFTPSYNC:-"false"}

########################################################################
# Read our config file
. "${BASEDIR}/etc/${NAME}.conf"

# Source our common functions
. "${BASEDIR}/etc/common"

# Set sane defaults if the configfile didn't do that for us.
# The directory for our logfiles
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
# Our own logfile
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
# Our lockfile directory
LOCKDIR=${LOCKDIR:-"${BASEDIR}/locks"}
# How many logfiles to keep
LOGROTATE=${LOGROTATE:-14}
# Our mirrorfile
MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"}
# used by log()
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
# extra ssh options we might want hostwide
SSH_OPTS=${SSH_OPTS:-"-o StrictHostKeyChecking=no"}
# Whats our archive name? We will also tell our leafs about it
PUSHARCHIVE=${PUSHARCHIVE:-"${CONF}"}
# How long to wait for mirrors to do stage1 if we have multi-stage syncing
PUSHDELAY=${PUSHDELAY:-600}
# Which ssh key to use?
KEYFILE=${KEYFILE:-".ssh/pushmirror"}
# where to send mails to
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
	# We are not on a debian.org host
	MAILTO=${MAILTO:-"root"}
else
	# Yay, on a .debian.org host
	MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi

if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
	error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG}
	exit 5
fi

# Hooks
HOOK1=${HOOK1:-""}
HOOK2=${HOOK2:-""}
HOOK3=${HOOK3:-""}

########################################################################

# Some sane defaults
cd ${BASEDIR}
umask 022

# Make sure we have our log and lock directories
mkdir -p "${LOGDIR}"
mkdir -p "${LOCKDIR}"

trap 'log "Mirrorpush done" >> ${LOG}; savelog "${LOG}" > /dev/null' EXIT

log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> ${LOG}

HOOK=(
	HOOKNR=1
	HOOKSCR=${HOOK1}
)
hook $HOOK

# From here on we do *NOT* want to exit on errors. We don't want to
# stop pushing mirrors just because we can't reach one of them.
set +e

# Built up our list of 2-stage mirrors.
PUSHLOCKS=""
PUSHLOCKS=$(get2stage)

# In case we have it - remove. It is used to synchronize multi-stage mirroring
rm -f "${LOCKDIR}/all_stage1"

# Now read our mirrorfile and push the mirrors defined in there.
# We use grep to easily sort out all lines having a # in front of them or are empty.
egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do
	if [ "x${MTYPE}x" = "xDELAYx" ]; then
		# We should wait a bit.
		if [ -z ${MLNAME} ]; then
			MLNAME=600
		fi
		log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}"
		sleep ${MLNAME}
		continue
	fi

    # If we are told we have a mhop sync to do and are called from within ftpsync,
    # we will only look at staged/mhop entries and ignore the rest.
    if [ "x${PUSHKIND}x" = "xmhopx" ] && [ "x${FROMFTPSYNC}x" = "xtruex" ]; then
        if [ "x${MTYPE}x" != "xstagedx" ] && [ "x${MTYPE}x" != "xmhopx" ]; then
            continue
        fi
    fi

    # Now, MSSHOPT may start with a -. In that case the whole rest of the line is taken
    # as a set of options to give to ssh, we pass it without doing anything with it.
    # If it starts with a 1 or 2 then it will tell us about the ssh protocol version to use,
    # and also means we look if there is one value more after a space. That value would then
    # be the ssh keyfile we use with -i. That gives us full flexibility for all
    # ssh options but doesn't destroy backwards compatibility.
    # If it is empty we assume proto 2 and the default keyfile.
    #
    # There is one bug in here. We will give out the master keyfile, even if there is a
    # "-i /bla/bla" in the options. ssh stuffs them together and presents two keys to the
    # target server. In the case both keys do some actions- the first one presented wins.
    # And this might not be what one wants.
    #
    # The only sane way to go around this, i think, is by dropping backward compability.
    # Which I don't really like.
    if [ -n "${MSSHOPT}" ]; then
        # So its not empty, lets check if it starts with a - and as such is a "new-style"
        # ssh options set.
        if [ "x${MSSHOPT:0:1}x" = "x-x" ]; then
            # Yes we start with a -
            SSHOPT="${MSSHOPT}"
            MPROTO="99"
            MKEYFILE="${BASEDIR}/${KEYFILE}"
        elif [ ${MSSHOPT:0:1} -eq 1 ] || [ ${MSSHOPT:0:1} -eq 2 ]; then
            # We do seem to have oldstyle options here.
            MPROTO=${MSSHOPT:0:1}
            MKEYFILE=${MSSHOPT:1}
            SSHOPT=""
        else
            error "I don't know what is configured for mirror ${MLNAME}"
            continue
        fi
    else
        MPROTO=2
		MKEYFILE="${BASEDIR}/${KEYFILE}"
        SSHOPT=""
    fi

	# Built our array
	SIGNAL_OPTS=(
		MIRROR="${MLNAME}"
		HOSTNAME="${MHOSTNAME}"
		USERNAME="${MUSER}"
		SSHPROTO="${MPROTO}"
		SSHKEY="${MKEYFILE}"
		SSHOPTS="${SSHOPT/ /#}"
		PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1"
		PUSHTYPE="${MTYPE}"
		PUSHARCHIVE=${PUSHARCHIVE}
		PUSHKIND=${PUSHKIND}
		FROMFTPSYNC=${FROMFTPSYNC}
	)

	# And finally, push the mirror
	log "Trigger ${MLNAME}" >> ${LOG}
	signal "${SIGNAL_OPTS}" &
	log "Trigger for ${MLNAME} done" >> ${LOG}

	HOOK=(
		HOOKNR=2
		HOOKSCR=${HOOK2}
	)
	hook $HOOK
	set +e
done

HOOK=(
	HOOKNR=3
	HOOKSCR=${HOOK3}
)
hook $HOOK

exit 0