1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
# -*- mode:sh -*-
# Little common functions
# push a mirror attached to us.
# Arguments (using an array named SIGNAL_OPTS):
#
# $MIRROR - Name for the mirror, also basename for the logfile
# $HOSTNAME - Hostname to push to
# $USERNAME - Username there
# $SSHPROTO - Protocol version, either 1 or 2.
# $SSHKEY - the ssh private key file to use for this push
# $SSHOPTS - any other option ssh accepts, passed blindly, be careful
# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged
# $PUSHTYPE - what kind of push should be done?
# all - normal, just push once with ssh backgrounded and finish
# staged - staged. first push stage1, then wait for $PUSHLOCKs to appear,
# then push stage2
# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!)
# $PUSHCB - do we want a callback?
#
# This function assumes that the variable LOG is set to a directory where
# logfiles can be written to.
# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings
# (list of "lock"files) to wait for if you want pushtype=staged
#
# Pushes might be done in background (for type all).
signal () {
ARGS="SIGNAL_OPTS[*]"
local ${!ARGS}
MIRROR=${MIRROR:-""}
HOSTNAME=${HOSTNAME:-""}
USERNAME=${USERNAME:-""}
SSHPROTO=${SSHPROTO:-""}
SSHKEY=${SSHKEY:-""}
SSHOPTS=${SSHOPTS:-""}
PUSHLOCKOWN=${PUSHLOCKOWN:-""}
PUSHTYPE=${PUSHTYPE:-"all"}
PUSHARCHIVE=${PUSHARCHIVE:-""}
PUSHCB=${PUSHCB:-""}
PUSHKIND=${PUSHKIND:-"all"}
FROMFTPSYNC=${FROMFTPSYNC:-"false"}
# And now get # back to space...
SSHOPTS=${SSHOPTS/\#/ }
# Defaults we always want, no matter what
SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no"
if [ -n "${SSH_OPTS}" ]; then
SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}"
fi
if [ -n "${SSHKEY}" ]; then
SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}"
fi
if [ -n "${SSHOPTS}" ]; then
SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}"
fi
if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ] && [ ${SSHPROTO} -ne 99 ]; then
# Idiots, we only want 1 or 2. Cant decide? Lets force 2.
SSHPROTO=2
fi
if [ -n "${SSHPROTO}" ] && [ ${SSHPROTO} -ne 99 ]; then
SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}"
fi
date -u >> "${LOGDIR}/${MIRROR}.log"
PUSHARGS=""
if [ -n "${PUSHARCHIVE}" ]; then
PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}"
fi
if [ -n "${PUSHCB}" ]; then
PUSHARGS="${PUSHARGS} sync:callback"
fi
if [ "xallx" = "x${PUSHTYPE}x" ]; then
# Default normal "fire and forget" push
echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log"
PUSHARGS1="sync:all"
ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1 &
elif [ "xstagedx" = "x${PUSHTYPE}x" ]; then
# Want a staged push. Fine, lets do that
echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log"
# Step1: Do a push to only sync stage1, do not background
PUSHARGS1="sync:stage1"
ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1
touch "${PUSHLOCKOWN}"
# Step2: Wait for all the other "lock"files to appear.
tries=0
# We do not wait forever
while [ ${tries} -lt ${PUSHDELAY} ]; do
total=0
found=0
for file in ${PUSHLOCKS}; do
total=$((total + 1))
if [ -f ${file} ]; then
found=$((found + 1))
fi
done
if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then
touch "${LOCKDIR}/all_stage1"
break
fi
tries=$((tries + 5))
sleep 5
done
# In case we did not have all PUSHLOCKS and still continued, note it
# This is a little racy, especially if the other parts decide to do this
# at the same time, but it wont hurt more than a mail too much, so I don't care much
if [ ${tries} -ge ${PUSHDELAY} ]; then
echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log"
for file in ${PUSHLOCKS}; do
if [ ! -f ${file} ]; then
echo "${file}" >> "${LOGDIR}/${MIRROR}.log"
error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing"
fi
done
fi
rm -f "${PUSHLOCKOWN}"
# Step3: It either timed out or we have all the "lock"files, sync stage2
PUSHARGS2="sync:stage2"
echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log"
ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1
else
# Can't decide? Then you get nothing.
return
fi
}
# callback, used by ftpsync
callback () {
# Defaults we always want, no matter what
SSH_OPTIONS="-o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no"
ssh $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME}
}
# log something (basically echo it together with a timestamp)
#
# Set $PROGRAM to a string to have it added to the output.
log () {
if [ -z "${PROGRAM}" ]; then
echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@"
else
echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@"
fi
}
# log the message using log() but then also send a mail
# to the address configured in MAILTO (if non-empty)
error () {
log "$@"
if [ -z "${MAILTO}" ]; then
echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO}
fi
}
# run a hook
# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR
# the script to run.
hook () {
ARGS='HOOK[@]'
local "${!ARGS}"
if [ -n "${HOOKSCR}" ]; then
log "Running hook $HOOKNR: ${HOOKSCR}"
set +e
${HOOKSCR}
result=$?
set -e
log "Back from hook $HOOKNR, got returncode ${result}"
return $result
else
return 0
fi
}
# Return the list of 2-stage mirrors.
get2stage() {
egrep '^(staged|mhop)' "${MIRRORS}" | {
while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do
PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}"
done
echo "$PUSHLOCKS"
}
}
# Rotate logfiles
savelog() {
torotate="$1"
count=${2:-${LOGROTATE}}
while [ ${count} -gt 0 ]; do
prev=$(( count - 1 ))
if [ -e "${torotate}.${prev}" ]; then
mv "${torotate}.${prev}" "${torotate}.${count}"
fi
count=$prev
done
mv "${torotate}" "${torotate}.0"
}
|
