summaryrefslogtreecommitdiff
path: root/bin/websync
blob: f46e3ecb2d09cf4156029e4c4dff24220462c51c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
#! /bin/bash
# No, we can not deal with sh alone.

set -e
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the function is
# the important part here)
set -E

# websync script for Debian
# Based losely on the old websync written by an
# unknown number of different people over the years and ftpsync.
#
# Copyright (C) 2008,2009 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# In case the admin somehow wants to have this script located someplace else,
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
# How the admin sets this isn't our place to deal with. One could use a wrapper
# for that. Or pam_env. Or whatever fits in the local setup. :)
BASEDIR=${BASEDIR:-"${HOME}"}

# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
# by Joerg Jaspert and the Debian mirroradm group.
# This is used to track which mirror is using which script version.
VERSION="0815"

# Source our common functions
. "${BASEDIR}/etc/common"

########################################################################
########################################################################
## functions                                                          ##
########################################################################
########################################################################
# All the stuff we want to do when we exit, no matter where
cleanup() {
    trap - ERR TERM HUP INT QUIT EXIT
    # all done. Mail the log, exit.
    log "Mirrorsync done";
    if [ -n "${MAILTO}" ]; then
        # In case rsync had something on stderr
        if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
            mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
        fi
        if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
            # And the normal log
            MAILFILES="${LOG}"
            if [ "x${FULLLOGS}x" = "xtruex" ]; then
                # Someone wants full logs including rsync
                MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
            fi
            cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
        fi
    fi

    savelog "${LOGDIR}/rsync-${NAME}.log"
    savelog "${LOGDIR}/rsync-${NAME}.error"
    savelog "$LOG" > /dev/null

    rm -f "${LOCK}"
}


# Check rsyncs return value
check_rsync() {

    ret=$1
    msg=$2

    # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
    # and us re-running. If it's not, uplink is broken anyways.
    case "${ret}" in
        0) return 0;;
        24) return 0;;
        23) return 2;;
        30) return 2;;
        *)
            error "ERROR: ${msg}"
            return 1
            ;;
    esac
}

########################################################################
########################################################################

# As what are we called?
NAME="`basename $0`"

# Now source the config.
. "${BASEDIR}/etc/${NAME}.conf"

########################################################################
# Config options go here. Feel free to overwrite them in the config    #
# file if you need to.                                                 #
# On debian.org machines the defaults should be ok.                    #
########################################################################

########################################################################
# There should be nothing to edit here, use the config file            #
########################################################################
MIRRORNAME=${MIRRORNAME:-`hostname -f`}
# Where to put logfiles in
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
# Our own logfile
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}

# Where should we put all the mirrored files?
TO=${TO:-"/org/www.debian.org/www"}

# used by log() and error()
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}

# Where to send mails about mirroring to?
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
    # We are not on a debian.org host
    MAILTO=${MAILTO:-"root"}
else
    # Yay, on a .debian.org host
    MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
# Want errors only or every log?
ERRORSONLY=${ERRORSONLY:-"true"}
# Want full logs, ie. including the rsync one?
FULLLOGS=${FULLLOGS:-"false"}

# How many logfiles to keep
LOGROTATE=${LOGROTATE:-14}

# Our lockfile
LOCK=${LOCK:-"${TO}/Website-Update-in-Progress-${MIRRORNAME}"}
# Do we need another rsync run?
UPDATEREQUIRED="${TO}/Website-Update-Required-${MIRRORNAME}"
# Trace file for mirror stats and checks (make sure we get full hostname)
TRACE=${TRACE:-".project/trace/${MIRRORNAME}"}

# rsync program
RSYNC=${RSYNC:-rsync}
# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
# excluded files
RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Website-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Website-Update-Required-${MIRRORNAME}"}
# Default rsync options for *every* rsync call
RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
# Which rsync share to use on our upstream mirror?
RSYNC_PATH=${RSYNC_PATH:-"web.debian.org"}

# our username for the rsync share
RSYNC_USER=${RSYNC_USER:-""}
# the password
RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}

# a possible proxy
RSYNC_PROXY=${RSYNC_PROXY:-""}

# General excludes.
EXCLUDE=${EXCLUDE:-"--exclude ${HOSTNAME}"}

# The temp directory used by rsync --delay-updates is not
# world-readable remotely. Always exclude it to avoid errors.
EXCLUDE="${EXCLUDE} --exclude .~tmp~/"

# And site specific excludes, by default its the sponsor stuff that should be local to all (except templates)
SITE_FILTER=${SITE_FILTER:-"--include sponsor.deb.* --exclude sponsor_img.* --exclude sponsor.html --exclude sponsor.*.html --filter=protect_sponsor_img.* --filter=protect_sponsor.html --filter=protect_sponsor.*.html"}

# Hooks
HOOK1=${HOOK1:-""}
HOOK2=${HOOK2:-""}
HOOK3=${HOOK3:-""}
HOOK4=${HOOK4:-""}

# Are we a hub?
HUB=${HUB:-"false"}

# Some sane defaults
cd "${BASEDIR}"
umask 022

# If we are here for the first time, create the
# destination and the trace directory
mkdir -p "${TO}/.project/trace"

# Used to make sure we will have the archive fully and completly synced before
# we stop, even if we get multiple pushes while this script is running.
# Otherwise we can end up with a half-synced archive:
# - get a push
# - sync, while locked
# - get another push. Of course no extra sync run then happens, we are locked.
# - done. Archive not correctly synced, we don't have all the changes from the second push.
touch "${UPDATEREQUIRED}"

# Check to see if another sync is in progress
if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
    if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
        # Process does either not exist or is not owned by us.
        echo "$$" > "${LOCK}"
    else
        echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
        exit 1
    fi
fi

trap cleanup EXIT ERR TERM HUP INT QUIT

# Start log by redirecting everything there.
exec >"$LOG" 2>&1 </dev/null

# Look who pushed us and note that in the log.
log "Mirrorsync start"
PUSHFROM="${SSH_CONNECTION%%\ *}"
if [ -n "${PUSHFROM}" ]; then
    log "We got pushed from ${PUSHFROM}"
fi
log "Acquired main lock"

HOOK=(
    HOOKNR=1
    HOOKSCR=${HOOK1}
)
hook $HOOK

# Now, we might want to sync from anonymous too.
# This is that deep in this script so hook1 could, if wanted, change things!
if [ -z ${RSYNC_USER} ]; then
    RSYNCPTH="${RSYNC_HOST}"
else
    RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
fi

# Now do the actual mirroring, and run as long as we have an updaterequired file.
export RSYNC_PASSWORD
export RSYNC_PROXY

while [ -e "${UPDATEREQUIRED}" ]; do
    log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"

    rm -f "${UPDATEREQUIRED}"
    log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER}  ${RSYNCPTH}::${RSYNC_PATH} ${TO}"

    set +e
    ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \
        ${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
    result=$?
    set -e

    log "Back from rsync with returncode ${result}"

    set +e
    check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}"
    GO=$?
    set -e

    if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
        log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
    elif [ ${GO} -ne 0 ]; then
        exit 3
    fi

    HOOK=(
        HOOKNR=2
        HOOKSCR=${HOOK2}
    )
    hook $HOOK

done

mkdir -p "${TO}/.project/trace"
LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
echo "Used websync version: ${VERSION}" >> "${TO}/${TRACE}"
echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"

HOOK=(
    HOOKNR=3
    HOOKSCR=${HOOK3}
)
hook $HOOK

if [ x${HUB} = "xtrue" ]; then
    log "Trigger slave mirrors"
    ${BASEDIR}/bin/runmirrors "websync"
    log "Trigger slave done"

    HOOK=(
        HOOKNR=4
        HOOKSCR=${HOOK4}
    )
    hook $HOOK
fi

# All done, rest is done by cleanup hook.