From c4cbd290c89d38687dc4f523fb8b44851066975d Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Mon, 11 Apr 2005 21:16:38 +0000 Subject: Add tor and mixminion --- other/mixminion/Makefile | 21 ++ other/mixminion/bin/rrd-graph | 103 ++++++++++ other/mixminion/bin/rrd-update | 147 ++++++++++++++ other/tor/Makefile | 25 +++ other/tor/bin/rrd-graph | 413 +++++++++++++++++++++++++++++++++++++ other/tor/bin/rrd-update | 448 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 1157 insertions(+) create mode 100644 other/mixminion/Makefile create mode 100755 other/mixminion/bin/rrd-graph create mode 100755 other/mixminion/bin/rrd-update create mode 100644 other/tor/Makefile create mode 100755 other/tor/bin/rrd-graph create mode 100755 other/tor/bin/rrd-update (limited to 'other') diff --git a/other/mixminion/Makefile b/other/mixminion/Makefile new file mode 100644 index 0000000..78706a5 --- /dev/null +++ b/other/mixminion/Makefile @@ -0,0 +1,21 @@ +TODAY=`TZ=UTC date +'%Y-%m-%d_%H:%M:%S'` +WGET=wget --cache=off +OUTDIR=mixminion-directory +SOURCE=http://mixminion.net/directory/Directory.gz +TMPFILE=download.tmp +LAST_NAME=dir.last + +all: + rm -f $(TMPFILE) + $(WGET) -q -O $(TMPFILE) $(SOURCE) + NAME=directory-$(TODAY).gz; \ + if [ -e $(LAST_NAME) ] && diff -q $(TMPFILE) $(OUTDIR)/`cat $(LAST_NAME)` >/dev/null ; then \ + ln $(OUTDIR)/`cat $(LAST_NAME)` $(OUTDIR)/$$NAME; \ + else \ + mv $(TMPFILE) $(OUTDIR)/$$NAME; \ + echo "$$NAME" > $(LAST_NAME); \ + fi; \ + bin/rrd-update $$NAME + rm -rf result + mkdir result + bin/rrd-graph diff --git a/other/mixminion/bin/rrd-graph b/other/mixminion/bin/rrd-graph new file mode 100755 index 0000000..269a2ad --- /dev/null +++ b/other/mixminion/bin/rrd-graph @@ -0,0 +1,103 @@ +#!/usr/bin/perl -w + +use strict; +use RRDs; +use BER; + +my $HOSTNAME = `hostname`; + +#my $IMG_URL = 'http://images.noreply.org/mixminion-nodes/'; +my $IMG_URL = ''; +my $NOW = time; +my $GMTIME = gmtime($NOW); +my $START_AT = $NOW-9*30*24*3600; +my $RRD = '/home/weasel/www/www.noreply.org/Build/other/mixminion/rrd/nodes.rrd'; +my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/mixminion/rrd/nodes'; + +opendir(D,$RRD_DIR) or die ("Cannot opendir .: $!\n"); +my @SOURCES = sort { lc($a) cmp lc($b)} grep { /\.rrd$/ } readdir(D); +@SOURCES = grep { + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, + $atime,$mtime,$ctime,$blksize,$blocks) = stat($RRD_DIR.'/'.$_); + $mtime >= $START_AT; + } @SOURCES; +closedir(D); + +my @params = qw{result/total.png}; +push @params, sprintf('-s %s', $START_AT); +#push @params, sprintf('-e %s', $NOW- 2*24*3600); +push @params, sprintf('-t %s', "Mixminion nodes"); +push @params, sprintf('-v %s', "# nodes"); +#push @params, sprintf('-w %d', 600); +#push @params, sprintf('-h %d', 400); + +push @params, sprintf('DEF:%s=%s:%s:%s', 'd_inDirectory' , $RRD, 'inDirectory', 'AVERAGE'); +push @params, sprintf('CDEF:%s=%s' , 'c_inDirectory' , 'd_inDirectory'); +push @params, sprintf('%s:%s#%s:%s' , 'LINE2' , 'c_inDirectory' , 'FF77FF', '# in directory'); + +push @params, sprintf('DEF:%s=%s:%s:%s', 'd_recommended' , $RRD, 'recommended', 'AVERAGE'); +push @params, sprintf('CDEF:%s=%s' , 'c_recommended' , 'd_recommended'); +push @params, sprintf('%s:%s#%s:%s' , 'LINE3' , 'c_recommended' , '7777FF', '# recommended' ); + +push @params, sprintf('CDEF:%s=d_inDirectory,0,*,%s,+' , 'zeroline' , '0'); +push @params, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); + + +my $body; +my $index; + +for my $source (@SOURCES) { + my $def = $source; + $def =~ s/\.rrd$//; + my $cdef = 'c_'.$def; + my $label = $def; + + $source = $RRD_DIR .'/'. $source; + + my @this; + push @this, sprintf('result/node-%s.png', $def); + push @this, sprintf('-s %s', $START_AT); + push @this, sprintf('-t %s', "$def"); + push @this, sprintf('-v %s', "#"); + #push @this, sprintf('-w %d', 150); + #push @this, sprintf('-h %d', 50); + + push @this, sprintf('DEF:%s=%s:%s:%s', 'd_inDirectory' , $source, 'inDirectory', 'AVERAGE'); + push @this, sprintf('CDEF:%s=%s' , 'c_inDirectory' , 'd_inDirectory'); + push @this, sprintf('%s:%s#%s:%s' , 'AREA' , 'c_inDirectory' , 'FF77FF', '# in directory'); + + push @this, sprintf('DEF:%s=%s:%s:%s', 'd_recommended' , $source, 'recommended', 'AVERAGE'); + push @this, sprintf('CDEF:%s=%s' , 'c_recommended' , 'd_recommended'); + push @this, sprintf('%s:%s#%s:%s' , 'AREA' , 'c_recommended' , '7777FF', '# recommended' ); + + push @this, sprintf('CDEF:%s=d_inDirectory,0,*,%s,+' , 'zeroline' , '0'); + push @this, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); + + + my ($averages,$xsize,$ysize) = RRDs::graph @this; + warn RRDs::error if RRDs::error; + + $body .= "

$def

top

\"$def

\n"; + $index .= "
  • $def
    • \n"; +} + +my ($averages,$xsize,$ysize) = RRDs::graph @params; +die RRDs::error if RRDs::error; + +my $html = "Number of Mixminion Nodes"; +$html .= "

    Number of Mixminion Nodes

    "; +$html .= "..
    \n"; +$html .= "

    "; +$html .= "

    Nodes

    "; +$html .= ""; +$html .= $body; +$html .= "


    \n"; +$html .= 'Images created with Tobi Oetiker\'s rrdtool'."\n"; +$html .= "
    \nBuilt at $GMTIME on $HOSTNAME
    \n"; +$html .= "Peter Palfrader <web\@palfrader.org>\n"; +$html .= "\n"; + +open (F, ">result/index.html") or die ("Cannot open index.html: $!\n"); +print F $html; +close F; + diff --git a/other/mixminion/bin/rrd-update b/other/mixminion/bin/rrd-update new file mode 100755 index 0000000..29d0c17 --- /dev/null +++ b/other/mixminion/bin/rrd-update @@ -0,0 +1,147 @@ +#!/usr/bin/perl -w + +use strict; +use RRDs; +use BER; +use Time::ParseDate; +use Compress::Zlib; + +my $NOW = time; +my $VERBOSE = 0; +my $RRD = '/home/weasel/www/www.noreply.org/Build/other/mixminion/rrd/nodes.rrd'; +my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/mixminion/rrd/nodes'; +my $DIR_DIR = '/home/weasel/www/www.noreply.org/Build/other/mixminion/mixminion-directory'; + +sub check_exists($) { + my ($rrd) = @_; + return if (-e $rrd); + my @params = ($rrd); + push @params, '-b', 'now - 1 year', + qw{ --step 14400 + DS:inDirectory:GAUGE:172800:U:U + DS:recommended:GAUGE:172800:U:U + RRA:AVERAGE:0.5:1:21900 + }; + print "Creating rrd: $rrd...\n" if $VERBOSE; + RRDs::create @params; + my $err=RRDs::error; + warn "ERROR while creating $rrd: $err\n" if $err; +} +sub get_last($) { + my ($rrd) = @_; + my $last = 0; + if ( -e $rrd) { + $last = RRDs::last($rrd); + my $err = RRDs::error; + warn "ERROR while getting last for $rrd: $err\n" if $err; + }; + return $last; +}; + +check_exists($RRD); +my $last = get_last($RRD); + +opendir(DIR, $RRD_DIR) || die ("Cannot opendir $RRD_DIR: $!\n"); +my @rrdfiles = grep { /\.rrd$/ } readdir (DIR); +closedir(DIR); +my %last_node; +for my $rrdfile (@rrdfiles) { + my $nodename = $rrdfile; + $nodename =~ s/\.rrd$//; + $last_node{$nodename} = get_last($RRD_DIR.'/'.$rrdfile); +} + +my @dirfiles; +if (scalar @ARGV) { + @dirfiles = @ARGV; +} else { + opendir(DIR, $DIR_DIR) || die ("Cannot opendir $DIR_DIR: $!\n"); + @dirfiles = sort { ($a cmp $b) } grep { /^directory-/ } readdir (DIR); + closedir(DIR); +}; + +my @updateGlobal; +my %updates; +for my $dir (@dirfiles) { + print "Doing $dir\n" if $VERBOSE; + open (DIRECTORY, $DIR_DIR.'/'.$dir) || die ("Cannot open $DIR_DIR/$dir: $!\n"); + my $compressed_dir = join '', ; + close (DIRECTORY); + + my $directory = Compress::Zlib::memGunzip($compressed_dir); + + my $published = undef; + my $recommended_servers = undef; + my %in_directory; + my %is_recommended; + my $section = '' ; + for my $line (split /\r?\n/, $directory) { + chomp $line; + if ($line =~ /^\[(.*)\]\s*$/) { + $section = $1; + }; + if ($section eq 'Directory' && $line =~ /^Published: (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/) { + my $date = $1; + print "Found published: $date\n" if ($VERBOSE >= 3); + $date =~ s#/#-#g; + $published = parsedate($date); + print "Parsed as $published (".(gmtime $published).")\n" if ($VERBOSE >= 4); + }; + if ($section eq 'Directory' && $line =~ m#^Published: (\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2})$#) { + my $date = $1; + print "Found published: $date\n" if ($VERBOSE >= 3); + $date =~ s#/#-#g; + $published = parsedate($date); + print "Parsed as $published (".(gmtime $published).")\n" if ($VERBOSE >= 4); + }; + if ($section eq 'Directory' && $line =~ /^Recommended-Servers: \s*(.*?)\s*$/) { + $recommended_servers = $1 + }; + if ($section eq 'Server' && $line =~ /Nickname: ([a-zA-Z0-9_-]+)\s*$/) { + $in_directory{$1} = 1; + }; + } + close (DIRECTORY); + my @recommended_servers = split /,?\s*\s+/, $recommended_servers; + for my $node (@recommended_servers) { + $is_recommended{$node} = 1; + } + + if ($published > $last) { + push @updateGlobal, $published.':'.(scalar keys %in_directory).':'.(scalar @recommended_servers); + $last = $published; + print "at ".(gmtime $published)." there are ".(scalar keys %in_directory).' in the directory and '.(scalar @recommended_servers)." recommended servers\n" if ($VERBOSE >= 3); + } else { + print "ignoring data at ".(gmtime $published)." because it's older than ".(gmtime $last)."\n" if ($VERBOSE >= 3); + }; + for my $node (keys %in_directory) { + if (!defined $last_node{$node} || $published > $last_node{$node}) { + push @{$updates{$node}}, $published.':1:'.( defined $is_recommended{$node} ? '1' : '0' ); + $last_node{$node} = $published; + } + delete $is_recommended{$node}; + } + if (scalar keys %is_recommended) { + my $nodes = join ', ', keys %is_recommended; + warn ("$nodes recommended but not in directory.\n"); + } +}; + +if (scalar @updateGlobal != 0) { + check_exists($RRD); + RRDs::update( $RRD, @updateGlobal ); + my $err=RRDs::error; + warn "ERROR while updating $RRD: $err\n" if $err; +}; + +for my $node (keys %updates) { + if ($node =~ /[^a-zA-Z0-9_-]/) { + warn ("Illegal characters in node name '$node'\n"); + next; + }; + my $rrd = $RRD_DIR.'/'.$node.'.rrd'; + check_exists($rrd); + RRDs::update( $rrd, @{$updates{$node}} ); + my $err=RRDs::error; + warn "ERROR while updating $rrd: $err\n" if $err; +}; diff --git a/other/tor/Makefile b/other/tor/Makefile new file mode 100644 index 0000000..e08ee8e --- /dev/null +++ b/other/tor/Makefile @@ -0,0 +1,25 @@ +TODAY=`TZ=UTC date +'%Y-%m-%d_%H:%M:%S'` +WGET=wget --cache=off +OUTDIR=tor-directory +#SOURCE=http://moria.mit.edu:9031/ +SOURCE=http://tor.noreply.org:9030/ +TMPFILE=download.tmp +LAST_NAME=dir.last + +all: + rm -f $(TMPFILE) + $(WGET) -q -O $(TMPFILE) $(SOURCE) + NAME=directory-$(TODAY); \ + if [ -e $(LAST_NAME) ] && diff -q $(TMPFILE) $(OUTDIR)/`cat $(LAST_NAME)` >/dev/null ; then \ + ln $(OUTDIR)/`cat $(LAST_NAME)` $(OUTDIR)/$$NAME; \ + else \ + mv $(TMPFILE) $(OUTDIR)/$$NAME; \ + echo "$$NAME" > $(LAST_NAME); \ + fi; \ + bin/rrd-update $$NAME + rm -rf result.new + mkdir result.new + bin/rrd-graph + mv result result.old + mv result.new result + rm -rf result.old diff --git a/other/tor/bin/rrd-graph b/other/tor/bin/rrd-graph new file mode 100755 index 0000000..63c5c43 --- /dev/null +++ b/other/tor/bin/rrd-graph @@ -0,0 +1,413 @@ +#!/usr/bin/perl -w + +use strict; +use RRDs; +use BER; + +my $HOSTNAME = `hostname`; + +#my $IMG_URL = 'http://images.noreply.org/tor-running-routers/'; +#my $IMG_URL = 'http://new.noreply.org/tor-running-routers/'; +my $IMG_URL = ''; +my $NOW = time; +my $GMTIME = gmtime($NOW); +my $START_AT = $NOW-4*30*24*3600; +my $END_AT = $NOW-4*3600; +my $OLD_AT = $NOW-7*24*3600; +my $RRD = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/running_Routers.rrd'; +my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes'; +my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index'; +my $RESULT_DIR = 'result.new'; + +# Get list of RRD files +opendir(D,$RRD_DIR) or die ("Cannot opendir $RRD_DIR: $!\n"); +my @SOURCES = grep { /\.rrd$/ } readdir(D); +@SOURCES = grep { + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, + $atime,$mtime,$ctime,$blksize,$blocks) = stat($RRD_DIR.'/'.$_); + $mtime >= $START_AT; + } @SOURCES; +closedir(D); + +# Read hash <-> name mapping +my %hashes_router; +opendir(D,$INDEX_DIR) or die ("Cannot opendir $INDEX_DIR: $!\n"); +my @NAME = grep { /\.name$/ } readdir(D); +closedir(D); +for my $hash (@NAME) { + my $f = $INDEX_DIR.'/'.$hash; + open (F, $f) or warn("Cannot open $f: $!\n"), next; + my $name = ; + my $platform = ; + my $published = ; + chomp $name; + chomp $platform; + chomp $published; + close F; + $hash =~ s/\.name$//; + $hashes_router{$hash} = {name => $name, platform => $platform, published => $published }; +}; + +@SOURCES = grep { !/^TRAF-/ } @SOURCES; +@SOURCES = sort { + my $aa = $a; + my $bb = $b; + $aa =~ s/\.rrd//; + $bb =~ s/\.rrd//; + lc($hashes_router{$aa}->{'name'}) cmp lc($hashes_router{$bb}->{'name'}) || + $a cmp $b + } @SOURCES; + +my @params = 'MAGIX1'; +push @params, sprintf('MAGIX2'); +push @params, sprintf('-e %s', $END_AT); +#push @params, sprintf('-e %s', $NOW- 2*24*3600); +push @params, sprintf('-t %s', "Running routers"); +push @params, sprintf('-aPNG'); +push @params, sprintf('-v %s', "# Routers"); +#push @params, sprintf('-w %d', 600); +#push @params, sprintf('-h %d', 400); +push @params, sprintf('-w %d', 497); +push @params, sprintf('-h %d', 301); +#push @params, sprintf('-h %d', 271); + +push @params, sprintf('DEF:%s=%s:%s:%s', 'd_runningVerified' , $RRD, 'runningVerified', 'AVERAGE'); +push @params, sprintf('CDEF:%s=%s' , 'c_runningVerified' , 'd_runningVerified'); +push @params, sprintf('DEF:%s=%s:%s:%s', 'd_runningUnverified' , $RRD, 'runningUnverified', 'AVERAGE'); +push @params, sprintf('CDEF:%s=%s' , 'c_runningUnverified' , 'd_runningUnverified'); +push @params, sprintf('DEF:%s=%s:%s:%s', 'd_exit80Verified' , $RRD, 'exit80Verified', 'AVERAGE'); +push @params, sprintf('CDEF:%s=%s' , 'c_exit80Verified' , 'd_exit80Verified'); + +push @params, sprintf('DEF:%s=%s:%s:%s', 'd_f_runningVerified' , $RRD, 'fastRunningVerified', 'AVERAGE'); +push @params, sprintf('CDEF:%s=%s' , 'c_f_runningVerified' , 'd_f_runningVerified'); +push @params, sprintf('DEF:%s=%s:%s:%s', 'd_f_runningUnverified' , $RRD, 'fastRunningUnverifi', 'AVERAGE'); +push @params, sprintf('CDEF:%s=%s' , 'c_f_runningUnverified' , 'd_f_runningUnverified'); +push @params, sprintf('DEF:%s=%s:%s:%s', 'd_f_exit80Verified' , $RRD, 'fastExit80Verified', 'AVERAGE'); +push @params, sprintf('CDEF:%s=%s' , 'c_f_exit80Verified' , 'd_f_exit80Verified'); + +#push @params, sprintf('%s:%s#%s:%s' , 'AREA' , 'c_runningVerified' , '7777FF', 'verified Nodes' ); +#push @params, sprintf('%s:%s#%s:%s' , 'STACK' , 'c_runningUnverified' , 'FF7700', 'unverified Nodes' ); +#push @params, sprintf('%s:%s#%s:%s' , 'LINE3' , 'c_exit80Verified' , '00FF00', 'verified Nodes exiting to port 80' ); +push @params, sprintf('%s:%s#%s:%s' , 'AREA' , 'c_runningVerified' , 'AAAAFF', 'verified Nodes' ); +push @params, sprintf('%s:%s#%s:%s' , 'STACK' , 'c_runningUnverified' , 'FF7700', '+unverified Nodes' ); +push @params, sprintf('%s:%s#%s:%s' , 'LINE2' , 'c_exit80Verified' , 'BBFFBB', 'verified Nodes exiting to port 80' ); + +push @params, sprintf('%s:%s#%s:%s' , 'LINE3' , 'c_f_runningVerified' , '0000FF', 'fast verified Nodes' ); +push @params, sprintf('%s:%s#%s:%s' , 'STACK' , 'c_f_runningUnverified' , 'FF0000', '+fast unverified Nodes' ); +push @params, sprintf('%s:%s#%s:%s' , 'LINE3' , 'c_f_exit80Verified' , '00FF00', 'fast verified Nodes exiting to port 80' ); + +push @params, sprintf('CDEF:%s=d_runningVerified,0,*,%s,+' , 'zeroline' , '0'); +push @params, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); + + + +my @totalTraffic = 'MAGIX1'; +push @totalTraffic, sprintf('MAGIX2'); +#push @totalTraffic, sprintf('-s %s', $START_AT); +push @totalTraffic, sprintf('-e %s', $END_AT); +#push @totalTraffic, sprintf('-e %s', $NOW- 2*24*3600); +push @totalTraffic, sprintf('-t %s', "Total Traffic"); +push @totalTraffic, sprintf('-aPNG'); +push @totalTraffic, sprintf('-v %s', "Bandwidth Used"); +#push @totalTraffic, sprintf('-w %d', 600); +#push @totalTraffic, sprintf('-h %d', 400); +push @totalTraffic, sprintf('-w %d', 497); +push @totalTraffic, sprintf('-h %d', 201); +#push @totalTraffic, sprintf('-h %d', 301); +#push @totalTraffic, sprintf('-h %d', 271); + + +my %body; +my %index; +my $traf_item_counter = 0; + +for my $source (@SOURCES) { + my $hash = $source; + $hash =~ s/\.rrd$//; + my $name = $hashes_router{$hash}->{'name'}; + my $platform = $hashes_router{$hash}->{'platform'}; + my $published = $hashes_router{$hash}->{'published'}; + my $def = $name; + my $cdef = 'c_'.$def; + my $label = $def; + + my $source_traffic = $RRD_DIR .'/TRAF-'. $source; + $source = $RRD_DIR .'/'. $source; + + my (undef,undef,undef,undef,undef,undef,undef,undef, undef,$mtime,undef,undef,undef) = stat($source); + my $section = $mtime < $OLD_AT ? 'old' : 'current'; + + + my @this; + push @this, sprintf($RESULT_DIR.'/node-%s.png', $hash); + push @this, sprintf('-s %s', $START_AT); + push @this, sprintf('-e %s', $END_AT); + push @this, sprintf('-t %s', "$name"); + push @this, sprintf('-aPNG'); + push @this, sprintf('-v %s', "running"); + push @this, sprintf('-w %d', 150); + push @this, sprintf('-h %d', 50); + + push @this, sprintf('DEF:%s=%s:%s:%s', 'd_runningVerified' , $source, 'runningVerified', 'AVERAGE'); + push @this, sprintf('CDEF:%s=%s' , 'c_runningVerified' , 'd_runningVerified'); + + push @this, sprintf('DEF:%s=%s:%s:%s', 'd_runningUnverified' , $source, 'runningUnverified', 'AVERAGE'); + push @this, sprintf('CDEF:%s=%s' , 'c_runningUnverified' , 'd_runningUnverified'); + + push @this, sprintf('DEF:%s=%s:%s:%s', 'd_exit80Verified' , $source, 'exit80Verified', 'AVERAGE'); + push @this, sprintf('CDEF:c_exit80Verified=d_exit80Verified,0.5,*,c_runningVerified,*'); + + push @this, sprintf('%s:%s#%s' , 'AREA' , 'c_runningVerified' , '7777FF' ); + push @this, sprintf('%s:%s#%s' , 'STACK' , 'c_runningUnverified' , 'FF7700' ); + push @this, sprintf('%s:%s#%s' , 'AREA' , 'c_exit80Verified' , '00FF00' ); + + push @this, sprintf('CDEF:%s=d_runningVerified,0,*,%s,+' , 'zeroline' , '0'); + push @this, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); + + my ($averages,$xsize,$ysize) = RRDs::graph @this; + warn RRDs::error if RRDs::error; + + $body{$section} .= "

    $name

    ". + "

    ". + "Last descriptor data:". + "

    ". + "platform $platform
    ". + "published $published". + "
    ". + "

    \"$name\n"; + $index{$section} .= "

  • $name
    • \n"; + + if (-e $source_traffic) { + $traf_item_counter++; + push @totalTraffic, sprintf('DEF:read%d=%s:read:AVERAGE', $traf_item_counter, $source_traffic); + push @totalTraffic, sprintf('DEF:write%d=%s:write:AVERAGE', $traf_item_counter, $source_traffic); + push @totalTraffic, sprintf('DEF:max_read%d=%s:read:MAX', $traf_item_counter, $source_traffic); + push @totalTraffic, sprintf('DEF:max_write%d=%s:write:MAX', $traf_item_counter, $source_traffic); + #push @totalTraffic, sprintf('DEF:min_read%d=%s:read:MIN', $traf_item_counter, $source_traffic); + #push @totalTraffic, sprintf('DEF:min_write%d=%s:write:MIN', $traf_item_counter, $source_traffic); + push @totalTraffic, sprintf('DEF:max_capacity%d=%s:capacity:MAX', $traf_item_counter, $source); + + push @totalTraffic, sprintf('CDEF:cleanread%d=read%d,UN,0,read%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); + push @totalTraffic, sprintf('CDEF:cleanwrite%d=write%d,UN,0,write%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); + push @totalTraffic, sprintf('CDEF:max_cleanread%d=max_read%d,UN,0,max_read%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); + push @totalTraffic, sprintf('CDEF:max_cleanwrite%d=max_write%d,UN,0,max_write%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); + #push @totalTraffic, sprintf('CDEF:min_cleanread%d=min_read%d,UN,0,min_read%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); + #push @totalTraffic, sprintf('CDEF:min_cleanwrite%d=min_write%d,UN,0,min_write%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); + push @totalTraffic, sprintf('CDEF:max_cleancapacity%d=max_capacity%d,UN,0,max_capacity%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); + + if ($traf_item_counter == 1) { + push @totalTraffic, sprintf('CDEF:aggread%d=cleanread%d', $traf_item_counter, $traf_item_counter); + push @totalTraffic, sprintf('CDEF:aggwrite%d=cleanwrite%d', $traf_item_counter, $traf_item_counter); + push @totalTraffic, sprintf('CDEF:max_aggread%d=max_cleanread%d', $traf_item_counter, $traf_item_counter); + push @totalTraffic, sprintf('CDEF:max_aggwrite%d=max_cleanwrite%d', $traf_item_counter, $traf_item_counter); + #push @totalTraffic, sprintf('CDEF:min_aggread%d=min_cleanread%d', $traf_item_counter, $traf_item_counter); + #push @totalTraffic, sprintf('CDEF:min_aggwrite%d=min_cleanwrite%d', $traf_item_counter, $traf_item_counter); + push @totalTraffic, sprintf('CDEF:max_aggcapacity%d=max_cleancapacity%d', $traf_item_counter, $traf_item_counter); + } else { + push @totalTraffic, sprintf('CDEF:aggread%d=cleanread%d,aggread%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); + push @totalTraffic, sprintf('CDEF:aggwrite%d=cleanwrite%d,aggwrite%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); + push @totalTraffic, sprintf('CDEF:max_aggread%d=max_cleanread%d,max_aggread%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); + push @totalTraffic, sprintf('CDEF:max_aggwrite%d=max_cleanwrite%d,max_aggwrite%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); + #push @totalTraffic, sprintf('CDEF:min_aggread%d=min_cleanread%d,min_aggread%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); + #push @totalTraffic, sprintf('CDEF:min_aggwrite%d=min_cleanwrite%d,min_aggwrite%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); + push @totalTraffic, sprintf('CDEF:max_aggcapacity%d=max_cleancapacity%d,max_aggcapacity%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); + }; + + for my $thumb (0, 1) { + @this = (); + push @this, sprintf($RESULT_DIR.'/node-'.($thumb ? 'small-' : '').'traf-%s.png', $hash); + #push @this, sprintf('-s %s', '-604800'); + push @this, sprintf('-s %s', $START_AT); + push @this, sprintf('-e %s', $END_AT); + push @this, sprintf('-t %s', "$name traffic"); + push @this, sprintf('-v %s', "bytes/sec"); + if ($thumb) { + push @this, sprintf('-w %d', 150); + push @this, sprintf('-h %d', 50); + }; + push @this, sprintf('-aPNG'); + push @this, sprintf('DEF:read=%s:read:AVERAGE', $source_traffic); + push @this, sprintf('DEF:maxread=%s:read:MAX', $source_traffic); + #push @this, sprintf('DEF:minread=%s:read:MIN', $source_traffic); + push @this, sprintf('DEF:write=%s:write:AVERAGE', $source_traffic); + push @this, sprintf('DEF:maxwrite=%s:write:MAX', $source_traffic); + #push @this, sprintf('DEF:minwrite=%s:write:MIN', $source_traffic); + push @this, sprintf('DEF:maxcapacity=%s:capacity:MAX', $source); + push @this, sprintf('AREA:read#00FF00:Read'); + push @this, sprintf('LINE2:write#0000FF:Write'); + unless ($thumb) { + push @this, sprintf('LINE1:maxread#009A2C:Read [max 15min]'); + push @this, sprintf('LINE1:maxwrite#5555FF:Write [max 15min]'); + #push @this, sprintf('LINE1:minread#FF76D4:Read [min]'); + #push @this, sprintf('LINE1:minwrite#222255:Write [min]'); + push @this, sprintf('LINE1:maxcapacity#FF5555:Capacity'); + } + #push @this, sprintf('COMMENT:\\n'); + #push @this, sprintf('COMMENT:%s\\r', $GMTIME); + #push @this, sprintf('COMMENT:\\n'); + #push @this, sprintf('COMMENT: Min Max Average Last\\n'); + #push @this, sprintf('COMMENT:Read: '); + #push @this, sprintf('GPRINT:minread:MIN:%6.2lf %sb/s '); + #push @this, sprintf('GPRINT:maxread:MAX:%6.2lf %sb/s '); + #push @this, sprintf('GPRINT:read:AVERAGE:%6.2lf %sb/s '); + #push @this, sprintf('GPRINT:read:LAST:%6.2lf %sb/s\\n'); + #push @this, sprintf('COMMENT:Write: '); + #push @this, sprintf('GPRINT:minwrite:MIN:%6.2lf %sb/s '); + #push @this, sprintf('GPRINT:maxwrite:MAX:%6.2lf %sb/s '); + #push @this, sprintf('GPRINT:write:AVERAGE:%6.2lf %sb/s '); + #push @this, sprintf('GPRINT:write:LAST:%6.2lf %sb/s\\n'); + + #push @this, sprintf('CDEF:%s=read,0,*,%s,+' , 'zeroline' , '0'); + #push @this, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); + + ($averages,$xsize,$ysize) = RRDs::graph @this; + warn RRDs::error if RRDs::error; + + }; + $body{$section} .= " "; + $body{$section} .= "\"$name"; + $body{$section} .= ""; + }; + #$body{$section} .= "
    top"; + $body{$section} .= "
    back"; + $body{$section} .= '

    '; +}; +push @totalTraffic, sprintf('AREA:aggread%d#00FF00:Read bytes/s', $traf_item_counter); +push @totalTraffic, sprintf('LINE2:aggwrite%d#0000FF:Write bytes/s', $traf_item_counter); +push @totalTraffic, sprintf('LINE1:max_aggread%d#009A2C:Read [max 15min]', $traf_item_counter); +push @totalTraffic, sprintf('LINE1:max_aggwrite%d#5555FF:Write [max 15min]', $traf_item_counter); +#push @totalTraffic, sprintf('LINE1:min_aggread%d#FF76D4:Read [min]', $traf_item_counter); +#push @totalTraffic, sprintf('LINE1:min_aggwrite%d#222255:Write [min]', $traf_item_counter); +push @totalTraffic, sprintf('LINE1:max_aggcapacity%d#FF5555:Capacity', $traf_item_counter); + +push @totalTraffic, sprintf('CDEF:zeroline=aggread%d,0,*', $traf_item_counter); +push @totalTraffic, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); + + +my $html = "Number of Running Tor routers"; +$html .= "

    Number of Running Tor routers

    "; +$html .= "..
    \n"; + + +if ($params[0] ne 'MAGIX1' || $params[1] ne 'MAGIX2') { + warn "params[0,1] aren't MAGIX1,2. not nice"; +} else { + $params[0] = $RESULT_DIR.'/totalWeekly.png'; + $params[1] = sprintf('-s %s', $NOW-7*24*3600); + my ($averages,$xsize,$ysize) = RRDs::graph @params; + warn RRDs::error if RRDs::error; + + $params[0] = $RESULT_DIR.'/totalBiWeekly.png'; + $params[1] = sprintf('-s %s', $NOW-14*24*3600); + ($averages,$xsize,$ysize) = RRDs::graph @params; + warn RRDs::error if RRDs::error; + + $params[0] = $RESULT_DIR.'/totalMonthly.png'; + $params[1] = sprintf('-s %s', $NOW-35*24*3600); + ($averages,$xsize,$ysize) = RRDs::graph @params; + warn RRDs::error if RRDs::error; + + $params[0] = $RESULT_DIR.'/total.png'; + $params[1] = sprintf('-s %s', $START_AT); + ($averages,$xsize,$ysize) = RRDs::graph @params; + warn RRDs::error if RRDs::error; + + $params[0] = $RESULT_DIR.'/total9Months.png'; + $params[1] = sprintf('-s %s', $NOW-9*30*24*3600); + ($averages,$xsize,$ysize) = RRDs::graph @params; + warn RRDs::error if RRDs::error; + + $html .= "

    "; + $html .= "

    \"Fast\" means a capacity of over 20,000 bytes/second

    \n"; + $html .= "

    Number of routers over the last 9 months | "; + $html .= " last month |"; + $html .= " last 2 weeks

    "; +}; + + +if ($totalTraffic[0] ne 'MAGIX1' || $totalTraffic[1] ne 'MAGIX2') { + warn "totalTraffic[0,1] aren't MAGIX1,2. not nice"; +} else { + $totalTraffic[0] = $RESULT_DIR.'/totalTrafficWeekly.png'; + $totalTraffic[1] = sprintf('-s %s', $NOW-7*24*3600); + my ($averages,$xsize,$ysize) = RRDs::graph @totalTraffic; + warn RRDs::error if RRDs::error; + + $totalTraffic[0] = $RESULT_DIR.'/totalTrafficBiWeekly.png'; + $totalTraffic[1] = sprintf('-s %s', $NOW-14*24*3600); + ($averages,$xsize,$ysize) = RRDs::graph @totalTraffic; + warn RRDs::error if RRDs::error; + + $totalTraffic[0] = $RESULT_DIR.'/totalTrafficMonthly.png'; + $totalTraffic[1] = sprintf('-s %s', $NOW-35*24*3600); + ($averages,$xsize,$ysize) = RRDs::graph @totalTraffic; + warn RRDs::error if RRDs::error; + + $totalTraffic[0] = $RESULT_DIR.'/totalTraffic.png'; + $totalTraffic[1] = sprintf('-s %s', $START_AT); + ($averages,$xsize,$ysize) = RRDs::graph @totalTraffic; + warn RRDs::error if RRDs::error; + + $totalTraffic[0] = $RESULT_DIR.'/totalTraffic9Months.png'; + $totalTraffic[1] = sprintf('-s %s', $NOW-9*30*24*3600); + ($averages,$xsize,$ysize) = RRDs::graph @totalTraffic; + warn RRDs::error if RRDs::error; + + $html .= "

    "; + $html .= "

    Bandwidth usage for last 9 months | "; + $html .= " last month |"; + $html .= " last 2 weeks

    "; +} + + + +$html .= "

    The directory is downloaded from "; + $html .= "http://tor.noreply.org:9030/"; + +$html .= " at regular intervals.
    \n"; +$html .= "Other directory sources are "; + $html .= "http://moria.mit.edu:9031/"; + +$html .= " and\n"; + $html .= "http://moria.mit.edu:9032/"; + +$html .= "."; +$html .= "

    Recent Nodes

    "; +$html .= "
      ".$index{'current'}."
    "; +#$html .= $body{'current'}; +$html .= "

    Older Nodes

    "; +$html .= "
      ".$index{'old'}."
    "; +#$html .= $body{'current'}; +$html .= "


    \n"; +$html .= 'Images created with Tobi Oetiker\'s rrdtool'."\n"; +$html .= "
    \nBuilt at $GMTIME on $HOSTNAME
    \n"; +$html .= "Peter Palfrader <web\@palfrader.org>\n"; +$html .= "\n"; + +open (F, ">$RESULT_DIR/index.html") or die ("Cannot open index.html: $!\n"); +print F $html; +close F; + +open (F, ">$RESULT_DIR/current.html") or die ("Cannot open current.html: $!\n"); +print F "Current Tor Routers", + "

    Current Tor routers

    ", + "./
    \n", + $body{'current'}, + "


    \n", + 'Images created with Tobi Oetiker\'s rrdtool'."\n", + "
    \nBuilt at $GMTIME on $HOSTNAME
    \n", + "Peter Palfrader <web\@palfrader.org>\n", + "\n"; +close F; + +open (F, ">$RESULT_DIR/old.html") or die ("Cannot open old.html: $!\n"); +print F "Older Tor Routers", + "

    Older Tor routers

    ", + "./
    \n", + $body{'old'}, + "


    \n", + 'Images created with Tobi Oetiker\'s rrdtool'."\n", + "
    \nBuilt at $GMTIME on $HOSTNAME
    \n", + "Peter Palfrader <web\@palfrader.org>\n", + "\n"; +close F; diff --git a/other/tor/bin/rrd-update b/other/tor/bin/rrd-update new file mode 100755 index 0000000..885eef3 --- /dev/null +++ b/other/tor/bin/rrd-update @@ -0,0 +1,448 @@ +#!/usr/bin/perl -w + +use strict; +use RRDs; +use BER; +use MIME::Base64; +use Digest::SHA1 qw(sha1_hex); +use Time::ParseDate; + +my $NOW = time; +my $VERBOSE = 1; +my $RRD = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/running_Routers.rrd'; +my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes'; +my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index'; +my $DIR_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/tor-directory'; +my $MIN_BANDWIDTH_FOR_FAST = 20000; + +sub check_exists_running($$) { + my ($rrd, $global) = @_; + return if (-e $rrd); + my @params = ($rrd); + push @params, '-b', 'now - 12 months', + qw{ --step 3600 + DS:runningVerified:GAUGE:172800:U:U + DS:runningUnverified:GAUGE:172800:U:U + DS:exit80Verified:GAUGE:172800:U:U }; + if ($global) { + push @params, qw{ + DS:fastRunningVerified:GAUGE:172800:U:U + DS:fastRunningUnverifi:GAUGE:172800:U:U + DS:fastExit80Verified:GAUGE:172800:U:U }; + } else { + push @params, 'DS:capacity:GAUGE:172800:U:U'; + }; + push @params, qw{ + RRA:MIN:0.5:1:336 + RRA:MAX:0.5:1:336 + RRA:AVERAGE:0.5:1:336 + RRA:LAST:0.5:1:336 + RRA:MIN:0.5:24:730 + RRA:MAX:0.5:24:730 + RRA:AVERAGE:0.5:24:730 + }; + print "Creating rrd: $rrd...\n" if $VERBOSE; + RRDs::create @params; + my $err=RRDs::error; + warn "ERROR while creating $rrd: $err\n" if $err; +} +sub check_exists_traffic($) { + my ($rrd) = @_; + return if (-e $rrd); + my @params = ($rrd); + push @params, '-b', 'now - 8 months', + qw{ --step 900 + DS:write:ABSOLUTE:900:U:U + DS:read:ABSOLUTE:900:U:U + RRA:AVERAGE:0.5:1:600 + RRA:AVERAGE:0.5:8:675 + RRA:AVERAGE:0.5:96:789 + RRA:MIN:0.5:1:600 + RRA:MIN:0.5:8:675 + RRA:MIN:0.5:96:789 + RRA:MAX:0.5:1:600 + RRA:MAX:0.5:8:675 + RRA:MAX:0.5:96:789 + RRA:LAST:0.5:1:600 + RRA:LAST:0.5:8:675 + RRA:LAST:0.5:96:789 + }; + print "Creating rrd: $rrd...\n" if $VERBOSE; + RRDs::create @params; + my $err=RRDs::error; + warn "ERROR while creating $rrd: $err\n" if $err; +} +sub get_last($) { + my ($rrd) = @_; + my $last = 0; + if ( -e $rrd) { + $last = RRDs::last($rrd); + my $err = RRDs::error; + warn "ERROR while getting last for $rrd: $err\n" if $err; + }; + return $last; +}; + +check_exists_running($RRD, 1); +my $last = get_last($RRD); + +opendir(DIR, $RRD_DIR) || die ("Cannot opendir $RRD_DIR: $!\n"); +my @rrdfiles = grep { /\.rrd$/ } readdir (DIR); +closedir(DIR); +my %last_node; +for my $rrdfile (@rrdfiles) { + my $nodename = $rrdfile; + $nodename =~ s/\.rrd$//; + $last_node{$nodename} = get_last($RRD_DIR.'/'.$rrdfile); +} + +my @dirfiles; +if (scalar @ARGV) { + @dirfiles = @ARGV; +} else { + opendir(DIR, $DIR_DIR) || die ("Cannot opendir $DIR_DIR: $!\n"); + @dirfiles = sort { ($a cmp $b) } grep { /^directory-/ } readdir (DIR); + closedir(DIR); +} + +my %KEY_TO_FPR_HASH = (); + +my %router_hashes; +my %hashes_router; +my @updateGlobal; +my %updates; +my %updates_traffic; +for my $dir (@dirfiles) { + print "Doing $dir...\n" if $VERBOSE; + open (DIRECTORY, $DIR_DIR.'/'.$dir) || die ("Cannot open $DIR_DIR/$dir: $!\n"); + # published 2004-06-11 02:07:01 + # recommended-software 0.0.6.2,0.0.7pre1-cvs-2,0.0.7pre1,0.0.7rc1,0.0.7rc1-cvs,0.0.7rc2,0.0.7 + # running-routers moria1 moria2 peacetime metacolo ovmja.... + my $dir_published = undef; + my $running_routers = undef; + my $linenr = 0; + while () { + $linenr++; + chomp; + if (/^published (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/) { + if (!defined $dir_published) { + $dir_published = parsedate($1); + } else { + warn ("dir_published already set to '$dir_published' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^running-routers\s*(.*?)\s*$/) { + if (!defined $running_routers) { + $running_routers = $1 + } else { + warn ("running_routers already set to '$running_routers' in directory $dir, line $linenr\n"); + next; + }; + }; + last if (/^\s*$/); + }; + + my $current_router = undef; + my $current_platform = undef; + my $current_published = undef; + my $current_capacity = undef; + my $current_hash = undef; + my %history; + my %router_desc; + my $current_write_history = undef; + my $current_read_history = undef; + my $current_port80 = undef; + while () { + $linenr++; + chomp; + if (/^router\s+(\S+)\s/) { + if (!defined $current_router) { + $current_router = $1; + } else { + warn ("current_router already set to '$current_router' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^platform\s+(.*)/) { + if (!defined $current_platform) { + $current_platform = $1; + } else { + warn ("current_platform already set to '$current_platform' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^published (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/) { + if (!defined $current_published) { + $current_published = $1; + } else { + warn ("current_published already set to '$current_published' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^bandwidth (\d+)\s+\d+\s+(\d+)$/) { + if (!defined $current_capacity) { + my $rate_limit = $1; + $current_capacity = $2; + $current_capacity = $rate_limit if ($rate_limit < $current_capacity); + } else { + warn ("current_capacity already set to '$current_capacity' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^opt write-history\s+(.*)/) { + if (!defined $current_write_history) { + $current_write_history = $1; + } else { + warn ("current_write_history already set to '$current_write_history' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^opt read-history\s+(.*)/) { + if (!defined $current_read_history) { + $current_read_history = $1; + } else { + warn ("current_read_history already set to '$current_read_history' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^(accept|reject) \*:(.*)/ && !defined $current_port80) { + my $policy = $1; + my $port = $2; + if ($port eq '*') { + $current_port80 = $policy eq 'accept'; + } elsif ($port =~ /^(\d+)$/) { + $current_port80 = $policy eq 'accept' if $port == 80; + } elsif ($port =~ /^(\d+)-(\d+)$/) { + my $min = $1; + my $max = $2; + $current_port80 = $policy eq 'accept' if $min <= 80 && 80 <= $max; + } else { + warn ("Cannot parse port spec '$port' in directory $dir, line $linenr\n"); + next; + } + }; + if (/^signing-key/) { + if (!defined $current_hash) { + if ($linenr++ && !~ /^-----BEGIN RSA PUBLIC KEY-----/) { + warn ("line after signing key not in expected form\n"); + $current_router = undef; + $current_platform = undef; + $current_published = undef; + $current_capacity = undef; + $current_port80 = undef; + next; + }; + my $key = ''; + my $line; + while ($linenr++ && ($line = )) { + last if ($line =~ /^-----END RSA PUBLIC KEY-----/); + $key .= $line; + } + if (!defined $line) { + warn ("No END RSA PUBLIC KEY in signing key of $current_router ($dir)\n"); + last; + }; + if (defined $KEY_TO_FPR_HASH{$key}) { + $current_hash = $KEY_TO_FPR_HASH{$key}; + } else { + $current_hash = uc(sha1_hex(decode_base64($key))); + $KEY_TO_FPR_HASH{$key} = $current_hash; + }; + } else { + warn ("current_hash already set to '$current_hash' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^\s*$/) { + if (!defined $current_router) { + warn("end of block without current_router\n"); + next; + }; + if (!defined $current_platform) { + warn("end of block without current_platform\n"); + next; + }; + if (!defined $current_published) { + warn("end of block without current_published\n"); + next; + }; + if (!defined $current_capacity) { + if ($current_platform =~ /^Tor 0\.0\.(6|7|8pre1-cvs)/) { + $current_capacity = 'U'; + } else { + warn("end of block without current_capacity in directory $dir, line $linenr\n"); + exit; + }; + }; + if (!defined $current_port80) { + warn("end of block without current_port80\n"); + next; + }; + if (!defined $current_hash) { + warn("end of block without current_hash\n"); + next; + }; + $router_hashes{$current_router} = $current_hash; + $hashes_router{$current_hash} = { name => $current_router, + platform => $current_platform, + published => $current_published }; + $router_desc{$current_hash} = { capacity => $current_capacity, + port80 => $current_port80 }; + $history{$current_hash} = { write_history => $current_write_history, + read_history => $current_read_history}; + $current_router = undef; + $current_platform = undef; + $current_published = undef; + $current_capacity = undef; + $current_port80 = undef; + $current_hash = undef; + $current_write_history = undef; + $current_read_history = undef; + } + } + close (DIRECTORY); + + # find out which nodes are running + my @running_routers = split /\s+/, $running_routers; + @running_routers = grep {! /^!/ } @running_routers; + my @verified_routers; + my @verified_routers_port80; + my @unverified_routers; + my @fast_verified_routers; + my @fast_verified_routers_port80; + my @fast_unverified_routers; + for my $router (@running_routers) { + my $hash; + if ($router =~ /^\$/) { + $hash = substr($router, 1); + } else { + $hash = $router_hashes{$router}; + if (!defined $hash) { + warn ("No hash known for $router\n"); + next; + }; + } + + if (!defined $router_desc{$hash}->{'port80'}) { + warn ("port 80 not defined for $router (this most likely means it was in running-routers but there was no server descriptor)\n"); + next; + }; + if (!defined $router_desc{$hash}->{'capacity'}) { + warn ("capacity not defined for $router (this most likely means it was in running-routers but there was no server descriptor)\n"); + next; + }; + + my $fast = 0; + if ($router_desc{$hash}->{'capacity'} eq 'U' || # in earlier times, all nodes where fast + $router_desc{$hash}->{'capacity'} >= $MIN_BANDWIDTH_FOR_FAST) { + $fast = 1; + }; + + + if ($router =~ /^\$/) { + push @unverified_routers, $hash; + push @fast_unverified_routers, $hash if $fast; + } else { + push @verified_routers, $hash; + push @verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'}); + + push @fast_verified_routers, $hash if $fast; + push @fast_verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'} && $fast); + } + + if (!defined $last_node{$hash} || $dir_published > $last_node{$hash}) { + push @{$updates{$hash}}, $dir_published.':'. + ($router =~ /^\$/ ? '0:1' : '1:0').':'. + $router_desc{$hash}->{'port80'}.':'. + $router_desc{$hash}->{'capacity'}; + $last_node{$hash} = $dir_published; + } + } + if ($dir_published > $last) { + push @updateGlobal, $dir_published.':'. + (scalar @verified_routers).':'. + (scalar @unverified_routers).':'. + (scalar @verified_routers_port80).':'. + (scalar @fast_verified_routers).':'. + (scalar @fast_unverified_routers).':'. + (scalar @fast_verified_routers_port80); + $last = $dir_published + }; + + # update traffic stats for all routers + for my $hash (keys %history) { + my %hist; + for my $inout (qw{write read}) { + my $hist = $history{$hash}->{$inout.'_history'}; + if (defined $hist) { + # 2004-09-26 01:45:45 (900 s) 9619165,12880477,10591411... + my ($time, $interval, $values) = + $hist =~ /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \((\d+) s\)\s?((?:\d+,)*\d+)?$/; + warn ("Could not parse history '$hist'\n"),next + unless defined $time; + my $stamp = parsedate($time); + warn ("Could not parse time in '$hist'\n"),next + unless defined $stamp; + next unless defined $values; + my @values = split /,/, $values; + for my $v (@values) { + $hist{$stamp}->{$inout} = $v; + $stamp -= $interval; + }; + }; + }; + for my $stamp (sort {$a <=> $b} keys %hist) { + if (!defined $last_node{'TRAF-'.$hash} || $stamp > $last_node{'TRAF-'.$hash}) { + push @{$updates_traffic{$hash}}, $stamp.':'. + (defined $hist{$stamp}->{'write'} ? $hist{$stamp}->{'write'} : '').':'. + (defined $hist{$stamp}->{'read'} ? $hist{$stamp}->{'read'} : ''); + $last_node{'TRAF-'.$hash} = $stamp; + }; + }; + }; +}; + +if (scalar @updateGlobal != 0) { + check_exists_running($RRD, 1); + RRDs::update( $RRD, @updateGlobal ); + my $err=RRDs::error; + warn "ERROR while updating $RRD: $err\n" if $err; +}; + +for my $router (keys %updates) { + next unless exists $hashes_router{$router}; + my $rrd = $RRD_DIR.'/'.$router.'.rrd'; + check_exists_running($rrd, 0); + RRDs::update( $rrd, @{$updates{$router}} ); + my $err=RRDs::error; + warn "ERROR while updating $rrd: $err\n" if $err; + my (($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)) = localtime($last_node{$router}); + my $stamp = sprintf("%04d%02d%02d%02d%02d.%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec); + print "Setting timestamp of $rrd to $stamp\n" if $VERBOSE; + system('touch', '-t', $stamp, $rrd); +}; + +for my $router (keys %updates_traffic) { + next unless exists $hashes_router{$router}; + my $rrd = $RRD_DIR.'/TRAF-'.$router.'.rrd'; + check_exists_traffic($rrd); + RRDs::update( $rrd, @{$updates_traffic{$router}} ); + my $err=RRDs::error; + warn "ERROR while updating $rrd: $err\n" if $err; + my (($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)) = localtime($last_node{'TRAF-'.$router}); + my $stamp = sprintf("%04d%02d%02d%02d%02d.%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec); + print "Setting timestamp of $rrd to $stamp\n" if $VERBOSE; + system('touch', '-t', $stamp, $rrd); +}; + + + +for my $hash (keys %hashes_router) { + my $f = $INDEX_DIR.'/'.$hash.".name"; + open(I, ">$f") or warn ("Cannot open $f: $!\n"), next; + print I $hashes_router{$hash}->{'name'},"\n"; + print I $hashes_router{$hash}->{'platform'},"\n"; + print I $hashes_router{$hash}->{'published'},"\n"; + close I; +}; -- cgit v1.2.3