diff options
Diffstat (limited to 'other/tor/bin/rrd-update')
| -rwxr-xr-x | other/tor/bin/rrd-update | 448 |
1 files changed, 448 insertions, 0 deletions
diff --git a/other/tor/bin/rrd-update b/other/tor/bin/rrd-update new file mode 100755 index 0000000..885eef3 --- /dev/null +++ b/other/tor/bin/rrd-update @@ -0,0 +1,448 @@ +#!/usr/bin/perl -w + +use strict; +use RRDs; +use BER; +use MIME::Base64; +use Digest::SHA1 qw(sha1_hex); +use Time::ParseDate; + +my $NOW = time; +my $VERBOSE = 1; +my $RRD = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/running_Routers.rrd'; +my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes'; +my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index'; +my $DIR_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/tor-directory'; +my $MIN_BANDWIDTH_FOR_FAST = 20000; + +sub check_exists_running($$) { + my ($rrd, $global) = @_; + return if (-e $rrd); + my @params = ($rrd); + push @params, '-b', 'now - 12 months', + qw{ --step 3600 + DS:runningVerified:GAUGE:172800:U:U + DS:runningUnverified:GAUGE:172800:U:U + DS:exit80Verified:GAUGE:172800:U:U }; + if ($global) { + push @params, qw{ + DS:fastRunningVerified:GAUGE:172800:U:U + DS:fastRunningUnverifi:GAUGE:172800:U:U + DS:fastExit80Verified:GAUGE:172800:U:U }; + } else { + push @params, 'DS:capacity:GAUGE:172800:U:U'; + }; + push @params, qw{ + RRA:MIN:0.5:1:336 + RRA:MAX:0.5:1:336 + RRA:AVERAGE:0.5:1:336 + RRA:LAST:0.5:1:336 + RRA:MIN:0.5:24:730 + RRA:MAX:0.5:24:730 + RRA:AVERAGE:0.5:24:730 + }; + print "Creating rrd: $rrd...\n" if $VERBOSE; + RRDs::create @params; + my $err=RRDs::error; + warn "ERROR while creating $rrd: $err\n" if $err; +} +sub check_exists_traffic($) { + my ($rrd) = @_; + return if (-e $rrd); + my @params = ($rrd); + push @params, '-b', 'now - 8 months', + qw{ --step 900 + DS:write:ABSOLUTE:900:U:U + DS:read:ABSOLUTE:900:U:U + RRA:AVERAGE:0.5:1:600 + RRA:AVERAGE:0.5:8:675 + RRA:AVERAGE:0.5:96:789 + RRA:MIN:0.5:1:600 + RRA:MIN:0.5:8:675 + RRA:MIN:0.5:96:789 + RRA:MAX:0.5:1:600 + RRA:MAX:0.5:8:675 + RRA:MAX:0.5:96:789 + RRA:LAST:0.5:1:600 + RRA:LAST:0.5:8:675 + RRA:LAST:0.5:96:789 + }; + print "Creating rrd: $rrd...\n" if $VERBOSE; + RRDs::create @params; + my $err=RRDs::error; + warn "ERROR while creating $rrd: $err\n" if $err; +} +sub get_last($) { + my ($rrd) = @_; + my $last = 0; + if ( -e $rrd) { + $last = RRDs::last($rrd); + my $err = RRDs::error; + warn "ERROR while getting last for $rrd: $err\n" if $err; + }; + return $last; +}; + +check_exists_running($RRD, 1); +my $last = get_last($RRD); + +opendir(DIR, $RRD_DIR) || die ("Cannot opendir $RRD_DIR: $!\n"); +my @rrdfiles = grep { /\.rrd$/ } readdir (DIR); +closedir(DIR); +my %last_node; +for my $rrdfile (@rrdfiles) { + my $nodename = $rrdfile; + $nodename =~ s/\.rrd$//; + $last_node{$nodename} = get_last($RRD_DIR.'/'.$rrdfile); +} + +my @dirfiles; +if (scalar @ARGV) { + @dirfiles = @ARGV; +} else { + opendir(DIR, $DIR_DIR) || die ("Cannot opendir $DIR_DIR: $!\n"); + @dirfiles = sort { ($a cmp $b) } grep { /^directory-/ } readdir (DIR); + closedir(DIR); +} + +my %KEY_TO_FPR_HASH = (); + +my %router_hashes; +my %hashes_router; +my @updateGlobal; +my %updates; +my %updates_traffic; +for my $dir (@dirfiles) { + print "Doing $dir...\n" if $VERBOSE; + open (DIRECTORY, $DIR_DIR.'/'.$dir) || die ("Cannot open $DIR_DIR/$dir: $!\n"); + # published 2004-06-11 02:07:01 + # recommended-software 0.0.6.2,0.0.7pre1-cvs-2,0.0.7pre1,0.0.7rc1,0.0.7rc1-cvs,0.0.7rc2,0.0.7 + # running-routers moria1 moria2 peacetime metacolo ovmja.... + my $dir_published = undef; + my $running_routers = undef; + my $linenr = 0; + while (<DIRECTORY>) { + $linenr++; + chomp; + if (/^published (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/) { + if (!defined $dir_published) { + $dir_published = parsedate($1); + } else { + warn ("dir_published already set to '$dir_published' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^running-routers\s*(.*?)\s*$/) { + if (!defined $running_routers) { + $running_routers = $1 + } else { + warn ("running_routers already set to '$running_routers' in directory $dir, line $linenr\n"); + next; + }; + }; + last if (/^\s*$/); + }; + + my $current_router = undef; + my $current_platform = undef; + my $current_published = undef; + my $current_capacity = undef; + my $current_hash = undef; + my %history; + my %router_desc; + my $current_write_history = undef; + my $current_read_history = undef; + my $current_port80 = undef; + while (<DIRECTORY>) { + $linenr++; + chomp; + if (/^router\s+(\S+)\s/) { + if (!defined $current_router) { + $current_router = $1; + } else { + warn ("current_router already set to '$current_router' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^platform\s+(.*)/) { + if (!defined $current_platform) { + $current_platform = $1; + } else { + warn ("current_platform already set to '$current_platform' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^published (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/) { + if (!defined $current_published) { + $current_published = $1; + } else { + warn ("current_published already set to '$current_published' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^bandwidth (\d+)\s+\d+\s+(\d+)$/) { + if (!defined $current_capacity) { + my $rate_limit = $1; + $current_capacity = $2; + $current_capacity = $rate_limit if ($rate_limit < $current_capacity); + } else { + warn ("current_capacity already set to '$current_capacity' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^opt write-history\s+(.*)/) { + if (!defined $current_write_history) { + $current_write_history = $1; + } else { + warn ("current_write_history already set to '$current_write_history' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^opt read-history\s+(.*)/) { + if (!defined $current_read_history) { + $current_read_history = $1; + } else { + warn ("current_read_history already set to '$current_read_history' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^(accept|reject) \*:(.*)/ && !defined $current_port80) { + my $policy = $1; + my $port = $2; + if ($port eq '*') { + $current_port80 = $policy eq 'accept'; + } elsif ($port =~ /^(\d+)$/) { + $current_port80 = $policy eq 'accept' if $port == 80; + } elsif ($port =~ /^(\d+)-(\d+)$/) { + my $min = $1; + my $max = $2; + $current_port80 = $policy eq 'accept' if $min <= 80 && 80 <= $max; + } else { + warn ("Cannot parse port spec '$port' in directory $dir, line $linenr\n"); + next; + } + }; + if (/^signing-key/) { + if (!defined $current_hash) { + if ($linenr++ && <DIRECTORY> !~ /^-----BEGIN RSA PUBLIC KEY-----/) { + warn ("line after signing key not in expected form\n"); + $current_router = undef; + $current_platform = undef; + $current_published = undef; + $current_capacity = undef; + $current_port80 = undef; + next; + }; + my $key = ''; + my $line; + while ($linenr++ && ($line = <DIRECTORY>)) { + last if ($line =~ /^-----END RSA PUBLIC KEY-----/); + $key .= $line; + } + if (!defined $line) { + warn ("No END RSA PUBLIC KEY in signing key of $current_router ($dir)\n"); + last; + }; + if (defined $KEY_TO_FPR_HASH{$key}) { + $current_hash = $KEY_TO_FPR_HASH{$key}; + } else { + $current_hash = uc(sha1_hex(decode_base64($key))); + $KEY_TO_FPR_HASH{$key} = $current_hash; + }; + } else { + warn ("current_hash already set to '$current_hash' in directory $dir, line $linenr\n"); + next; + }; + }; + if (/^\s*$/) { + if (!defined $current_router) { + warn("end of block without current_router\n"); + next; + }; + if (!defined $current_platform) { + warn("end of block without current_platform\n"); + next; + }; + if (!defined $current_published) { + warn("end of block without current_published\n"); + next; + }; + if (!defined $current_capacity) { + if ($current_platform =~ /^Tor 0\.0\.(6|7|8pre1-cvs)/) { + $current_capacity = 'U'; + } else { + warn("end of block without current_capacity in directory $dir, line $linenr\n"); + exit; + }; + }; + if (!defined $current_port80) { + warn("end of block without current_port80\n"); + next; + }; + if (!defined $current_hash) { + warn("end of block without current_hash\n"); + next; + }; + $router_hashes{$current_router} = $current_hash; + $hashes_router{$current_hash} = { name => $current_router, + platform => $current_platform, + published => $current_published }; + $router_desc{$current_hash} = { capacity => $current_capacity, + port80 => $current_port80 }; + $history{$current_hash} = { write_history => $current_write_history, + read_history => $current_read_history}; + $current_router = undef; + $current_platform = undef; + $current_published = undef; + $current_capacity = undef; + $current_port80 = undef; + $current_hash = undef; + $current_write_history = undef; + $current_read_history = undef; + } + } + close (DIRECTORY); + + # find out which nodes are running + my @running_routers = split /\s+/, $running_routers; + @running_routers = grep {! /^!/ } @running_routers; + my @verified_routers; + my @verified_routers_port80; + my @unverified_routers; + my @fast_verified_routers; + my @fast_verified_routers_port80; + my @fast_unverified_routers; + for my $router (@running_routers) { + my $hash; + if ($router =~ /^\$/) { + $hash = substr($router, 1); + } else { + $hash = $router_hashes{$router}; + if (!defined $hash) { + warn ("No hash known for $router\n"); + next; + }; + } + + if (!defined $router_desc{$hash}->{'port80'}) { + warn ("port 80 not defined for $router (this most likely means it was in running-routers but there was no server descriptor)\n"); + next; + }; + if (!defined $router_desc{$hash}->{'capacity'}) { + warn ("capacity not defined for $router (this most likely means it was in running-routers but there was no server descriptor)\n"); + next; + }; + + my $fast = 0; + if ($router_desc{$hash}->{'capacity'} eq 'U' || # in earlier times, all nodes where fast + $router_desc{$hash}->{'capacity'} >= $MIN_BANDWIDTH_FOR_FAST) { + $fast = 1; + }; + + + if ($router =~ /^\$/) { + push @unverified_routers, $hash; + push @fast_unverified_routers, $hash if $fast; + } else { + push @verified_routers, $hash; + push @verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'}); + + push @fast_verified_routers, $hash if $fast; + push @fast_verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'} && $fast); + } + + if (!defined $last_node{$hash} || $dir_published > $last_node{$hash}) { + push @{$updates{$hash}}, $dir_published.':'. + ($router =~ /^\$/ ? '0:1' : '1:0').':'. + $router_desc{$hash}->{'port80'}.':'. + $router_desc{$hash}->{'capacity'}; + $last_node{$hash} = $dir_published; + } + } + if ($dir_published > $last) { + push @updateGlobal, $dir_published.':'. + (scalar @verified_routers).':'. + (scalar @unverified_routers).':'. + (scalar @verified_routers_port80).':'. + (scalar @fast_verified_routers).':'. + (scalar @fast_unverified_routers).':'. + (scalar @fast_verified_routers_port80); + $last = $dir_published + }; + + # update traffic stats for all routers + for my $hash (keys %history) { + my %hist; + for my $inout (qw{write read}) { + my $hist = $history{$hash}->{$inout.'_history'}; + if (defined $hist) { + # 2004-09-26 01:45:45 (900 s) 9619165,12880477,10591411... + my ($time, $interval, $values) = + $hist =~ /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \((\d+) s\)\s?((?:\d+,)*\d+)?$/; + warn ("Could not parse history '$hist'\n"),next + unless defined $time; + my $stamp = parsedate($time); + warn ("Could not parse time in '$hist'\n"),next + unless defined $stamp; + next unless defined $values; + my @values = split /,/, $values; + for my $v (@values) { + $hist{$stamp}->{$inout} = $v; + $stamp -= $interval; + }; + }; + }; + for my $stamp (sort {$a <=> $b} keys %hist) { + if (!defined $last_node{'TRAF-'.$hash} || $stamp > $last_node{'TRAF-'.$hash}) { + push @{$updates_traffic{$hash}}, $stamp.':'. + (defined $hist{$stamp}->{'write'} ? $hist{$stamp}->{'write'} : '').':'. + (defined $hist{$stamp}->{'read'} ? $hist{$stamp}->{'read'} : ''); + $last_node{'TRAF-'.$hash} = $stamp; + }; + }; + }; +}; + +if (scalar @updateGlobal != 0) { + check_exists_running($RRD, 1); + RRDs::update( $RRD, @updateGlobal ); + my $err=RRDs::error; + warn "ERROR while updating $RRD: $err\n" if $err; +}; + +for my $router (keys %updates) { + next unless exists $hashes_router{$router}; + my $rrd = $RRD_DIR.'/'.$router.'.rrd'; + check_exists_running($rrd, 0); + RRDs::update( $rrd, @{$updates{$router}} ); + my $err=RRDs::error; + warn "ERROR while updating $rrd: $err\n" if $err; + my (($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)) = localtime($last_node{$router}); + my $stamp = sprintf("%04d%02d%02d%02d%02d.%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec); + print "Setting timestamp of $rrd to $stamp\n" if $VERBOSE; + system('touch', '-t', $stamp, $rrd); +}; + +for my $router (keys %updates_traffic) { + next unless exists $hashes_router{$router}; + my $rrd = $RRD_DIR.'/TRAF-'.$router.'.rrd'; + check_exists_traffic($rrd); + RRDs::update( $rrd, @{$updates_traffic{$router}} ); + my $err=RRDs::error; + warn "ERROR while updating $rrd: $err\n" if $err; + my (($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)) = localtime($last_node{'TRAF-'.$router}); + my $stamp = sprintf("%04d%02d%02d%02d%02d.%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec); + print "Setting timestamp of $rrd to $stamp\n" if $VERBOSE; + system('touch', '-t', $stamp, $rrd); +}; + + + +for my $hash (keys %hashes_router) { + my $f = $INDEX_DIR.'/'.$hash.".name"; + open(I, ">$f") or warn ("Cannot open $f: $!\n"), next; + print I $hashes_router{$hash}->{'name'},"\n"; + print I $hashes_router{$hash}->{'platform'},"\n"; + print I $hashes_router{$hash}->{'published'},"\n"; + close I; +}; |