1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
# -*- mode:sh -*-
# Little common functions
# push a mirror attached to us.
# Arguments (using an array named SIGNAL_OPTS):
#
# $MIRROR - Name for the mirror, also basename for the logfile
# $HOSTNAME - Hostname to push to
# $USERNAME - Username there
# $SSHPROTO - Protocol version, either 1 or 2.
# $SSHKEY - the ssh private key file to use for this push
# $SSHOPTS - any other option ssh accepts, passed blindly, be careful
# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged
# $PUSHTYPE - what kind of push should be done?
# all - normal, just push once with ssh backgrounded and finish
# staged - staged. first push stage1, then wait for $PUSHLOCKs to appear,
# then push stage2
# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!)
# $PUSHCB - do we want a callback?
#
# This function assumes that the variable LOG is set to a directory where
# logfiles can be written to.
# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings
# (list of "lock"files) to wait for if you want pushtype=staged
#
# Pushes might be done in background (for type all).
signal () {
ARGS="SIGNAL_OPTS[*]"
local ${!ARGS}
MIRROR=${MIRROR:-""}
HOSTNAME=${HOSTNAME:-""}
USERNAME=${USERNAME:-""}
SSHPROTO=${SSHPROTO:-""}
SSHKEY=${SSHKEY:-""}
SSHOPTS=${SSHOPTS:-""}
PUSHLOCKOWN=${PUSHLOCKOWN:-""}
PUSHTYPE=${PUSHTYPE:-"all"}
PUSHARCHIVE=${PUSHARCHIVE:-""}
PUSHCB=${PUSHCB:-""}
# Defaults we always want, no matter what
SSH_OPTIONS="-o BatchMode=yes -o SetupTimeOut=45 -o ConnectTimeout=45 -o PasswordAuthentication=no"
if [ -n "${SSH_OPTS}" ]; then
SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}"
fi
if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ]; then
# Idiots, we only want 1 or 2. Cant decide? Lets force 2
${SSHPROTO}=2
fi
date -u >> "${LOGDIR}/${MIRROR}.log"
if [ "xallx" = "x${PUSHTYPE}x" ]; then
# Default normal "fire and forget" push
ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "sync:all" >>"${LOGDIR}/${MIRROR}.log" 2>&1 &
elif [ "xstagedx" = "x${PUSHTYPE}x" ]; then
# Want a staged push. Fine, lets do that
PUSHARGS=""
if [ -n "${PUSHARCHIVE}" ]; then
PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}"
fi
if [ -n "${PUSHCB}" ]; then
PUSHARGS="${PUSHARGS} sync:callback"
fi
# Step1: Do a push to only sync stage1, do not background
PUSHARGS1="sync:stage1"
ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1
touch "${PUSHLOCKOWN}"
# Step2: Wait for all the other "lock"files to appear.
tries=0
# We do not wait forever
while [ ${tries} -lt ${PUSHDELAY} ]; do
total=0
found=0
for file in ${PUSHLOCKS}; do
total=$((total + 1))
if [ -f ${file} ]; then
found=$((found + 1))
fi
done
if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then
touch "${LOCKDIR}/all_stage1"
break
fi
sleep 5
done
rm -f "${PUSHLOCKOWN}"
# Step3: It either timed out or we have all the "lock"files, sync stage2
PUSHARGS2="sync:stage2"
ssh $SSH_OPTIONS -i "${SSHKEY}" -o"user ${USERNAME}" -${SSHPROTO} "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1
else
# Can't decide? Then you get nothing.
return
fi
}
# callback, used by ftpsync
callback () {
# Defaults we always want, no matter what
SSH_OPTIONS="-o BatchMode=yes -o SetupTimeOut=45 -o ConnectTimeout=45 -o PasswordAuthentication=no"
ssh $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME}
}
# log something (basically echo it together with a timestamp)
#
# Set $PROGRAM to a string to have it added to the output.
log () {
if [ -z "${PROGRAM}" ]; then
echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@"
else
echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@"
fi
}
# log the message using log() but then also send a mail
# to the address configured in MAILTO (if non-empty)
error () {
log "$@"
if [ -z "${MAILTO}" ]; then
echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO}
fi
}
# run a hook
# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR
# the script to run.
hook () {
ARGS='HOOK[@]'
local "${!ARGS}"
if [ -n "${HOOKSCR}" ]; then
log "Running hook $HOOKNR: ${HOOKSCR}"
set +e
${HOOKSCR}
result=$?
set -e
log "Back from hook $HOOKNR, got returncode ${result}"
return $result
else
return 0
fi
}
# Return the list of 2-stage mirrors.
get2stage() {
egrep '^staged' "${MIRRORS}" | {
while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do
PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}"
done
echo "$PUSHLOCKS"
}
}
# Rotate logfiles
savelog() {
torotate="$1"
count=${2:-${LOGROTATE}}
while [ ${count} -gt 0 ]; do
prev=$(( count - 1 ))
if [ -e "${torotate}.${prev}" ]; then
mv "${torotate}.${prev}" "${torotate}.${count}"
fi
count=$prev
done
mv "${torotate}" "${torotate}.0"
}
|
