From da50ee845fb50e9916e6f031783461fb62f68c47 Mon Sep 17 00:00:00 2001 From: Joerg Jaspert Date: Thu, 27 Nov 2008 20:50:18 +0100 Subject: pushpdo add a mirror script for packages.d.o --- bin/pushpdo | 112 ++++++++++++++++++++++++++++++++++++++++++++++ etc/pushpdo.conf.sample | 40 +++++++++++++++++ etc/pushpdo.mirror.sample | 21 +++++++++ 3 files changed, 173 insertions(+) create mode 100755 bin/pushpdo create mode 100644 etc/pushpdo.conf.sample create mode 100644 etc/pushpdo.mirror.sample diff --git a/bin/pushpdo b/bin/pushpdo new file mode 100755 index 0000000..30b0b37 --- /dev/null +++ b/bin/pushpdo @@ -0,0 +1,112 @@ +#! /bin/bash + +set -e +set -u + +# psuhpdo script for Debian +# +# Copyright (C) 2008 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +# In case the admin somehow wants to have this script located someplace else, +# he can set BASEDIR, and we will take that. If it is unset we take ${HOME} +BASEDIR=${BASEDIR:-"${HOME}"} + +NAME="`basename $0`" + + +# Read our config file +. "${BASEDIR}/etc/${NAME}.conf" + +# Source our common functions +. "${BASEDIR}/etc/common" + +# Set sane defaults if the configfile didn't do that for us. +# The directory for our logfiles +LOGDIR=${LOGDIR:-"${BASEDIR}/log"} +# Our own logfile +LOG=${LOG:-"${LOGDIR}/${NAME}.log"} +# How many logfiles to keep +LOGROTATE=${LOGROTATE:-14} +# Our mirrorfile +MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"} +# used by log() +PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"} +# extra ssh options we might want hostwide +SSH_OPTS=${SSH_OPTS:-""} +# Which ssh key to use? +KEYFILE=${KEYFILE:-".ssh/pushpackages"} +# which path to "mirror" +MIRRORPATH=${MIRRORPATH:-"/org/packages.debian.org/mirror/"} +# where to send mails to +if [ "x$(hostname -s)x" != "x${MIRRORNAME%%.debian.org}x" ]; then + # We are not on a debian.org host + MAILTO=${MAILTO:-"root"} +else + # Yay, on a .debian.org host + MAILTO=${MAILTO:-"mirrorlogs@debian.org"} +fi + +if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then + error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG} + exit 5 +fi + +# Some sane defaults +cd ${BASEDIR} +umask 022 + +# Make sure we have our log and lock directories +mkdir -p "${LOGDIR}" +mkdir -p "${LOCKDIR}" + +trap 'log "Pdopush done" >> ${LOG}; savelog "${LOG}" > /dev/null' EXIT + +log "Pushing pdo mirrors" >> ${LOG} + +# From here on we do *NOT* want to exit on errors. We don't want to +# stop pushing mirrors just because we can't reach one of them. +set +e + + +# Now read our mirrorfile and push the mirrors defined in there. +# We use grep to easily sort out all lines having a # in front of them or are empty. +egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" | +while read MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do + # Process the two options that can be left blank in the config + if [ -z ${MPROTO} ]; then + MPROTO=2 + fi + if [ -z ${MKEYFILE} ]; then + MKEYFILE="${BASEDIR}/${KEYFILE}" + fi + # Now, people can do stupid things and leave out the protocol, but + # define a keyfile... + if [ ${MPROTO} -ne 1 ] && [ ${MPROTO} -ne 2 ]; then + error "Need a correct ssh protocol version for ${MLNAME}, skipping" >> ${LOG} + continue + fi + + # And finally, push the mirror + log "Pushing ${MLNAME}" >> ${LOG} + # This needs a limited ssh key on the other side, something like + # no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="rsync --server -vlogDtpr . /srv/mirrors/packages.debian.org/",from="87.106.64.223,2001:8d8:80:11::35d,powell.debian.org" ssh-rsa..... + rsync -e "ssh -i ${MKEYFILE} -${MPROTO}" -av --progress --stats ${MIRRORPATH} ${MUSER}@${MHOSTNAME}:/does/not/matter + log "Pushing ${MLNAME} done" >> ${LOG} + + set +e +done + +exit 0 diff --git a/etc/pushpdo.conf.sample b/etc/pushpdo.conf.sample new file mode 100644 index 0000000..e9be9e3 --- /dev/null +++ b/etc/pushpdo.conf.sample @@ -0,0 +1,40 @@ +######################################################################## +######################################################################## +## This is a sample configuration file for the runmirror script. ## +## Most of the values are commented out and just shown here for ## +## completeness, together with their default value. ## +######################################################################## +######################################################################## + +## Which ssh key to use? +#KEYFILE=.ssh/pushmirror + +## The directory for our logfiles +#LOGDIR="${BASEDIR}/log" + +## Our own logfile +#LOG="${LOGDIR}/${NAME}.log" + +## Our lockfile directory +#LOCKDIR="${BASEDIR}/locks" + +## We do create a logfile for every run. To save space we rotate it, this +## defines how many we keep +#LOGROTATE=14 + +## Our mirrorfile +#MIRRORS="${BASEDIR}/etc/${NAME}.mirror" + +## extra ssh options we might want. *hostwide* +#SSH_OPTS="" + +## The script can send logs (or error messages) to a mail address. +## If this is unset it will default to the local root user unless it is run +## on a .debian.org machine where it will default to the mirroradm people. +#MAILTO="root" + +## How long to wait for mirrors to do stage1 if we have multi-stage syncing +#PUSHDELAY=240 + +## which path to push +#MIRRORPATH="/org/packages.debian.org/mirror/" diff --git a/etc/pushpdo.mirror.sample b/etc/pushpdo.mirror.sample new file mode 100644 index 0000000..711d496 --- /dev/null +++ b/etc/pushpdo.mirror.sample @@ -0,0 +1,21 @@ +# Definition of mirror hosts we push. +# One mirror per line, with the following fields defined. +# +# ShortName HostName User SSHProtocol SSHKeyFile +# +# ShortName will be used as a shorthand in logfile outputs and for the logfile +# where every ssh output gets redirected to. +# +# If no SSHKeyFile is given, the default from the config file applies. +# If SSHProtocol is empty, it will default to 2, but if you want to +# define a keyfile you HAVE TO set protocol too! +# +# Examples: +# +# piatti piatti.debian.org archvsync +# One special value is allowed: DELAY +# This word has to be on a line itself, followed by a space and a number. +# nothing else, not even whitespace. It will trigger a pause of $number +# seconds between the two mirrors. If no number is given it defaults to +# 60 seconds. +piatti piatti.debian.org archvsync -- cgit v1.2.3