#!/usr/bin/perl -w use strict; use RRDs; use MIME::Base64; use Digest::SHA1 qw(sha1_hex); use Time::ParseDate; my $NOW = time; my $VERBOSE = 1; my $RRD = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/running_Routers.rrd'; my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes'; my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index'; my $DIR_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/tor-directory'; my $MIN_BANDWIDTH_FOR_FAST = 20000; sub check_exists_running($$) { my ($rrd, $global) = @_; return if (-e $rrd); my @params = ($rrd); push @params, '-b', 'now - 24 months', qw{ --step 3600 DS:runningVerified:GAUGE:172800:U:U DS:runningUnverified:GAUGE:172800:U:U DS:exit80Verified:GAUGE:172800:U:U }; if ($global) { push @params, qw{ DS:fastRunningVerified:GAUGE:172800:U:U DS:fastRunningUnverifi:GAUGE:172800:U:U DS:fastExit80Verified:GAUGE:172800:U:U }; } else { push @params, 'DS:capacity:GAUGE:172800:U:U'; }; push @params, qw{ RRA:MIN:0.5:1:336 RRA:MAX:0.5:1:336 RRA:AVERAGE:0.5:1:336 RRA:LAST:0.5:1:336 RRA:MIN:0.5:24:730 RRA:MAX:0.5:24:730 RRA:AVERAGE:0.5:24:730 }; print "Creating rrd: $rrd...\n" if $VERBOSE; RRDs::create @params; my $err=RRDs::error; warn "ERROR while creating $rrd: $err\n" if $err; } sub check_exists_traffic($) { my ($rrd) = @_; return if (-e $rrd); my @params = ($rrd); push @params, '-b', 'now - 12 months', qw{ --step 900 DS:write:ABSOLUTE:900:U:U DS:read:ABSOLUTE:900:U:U RRA:AVERAGE:0.5:1:600 RRA:AVERAGE:0.5:8:675 RRA:AVERAGE:0.5:96:789 RRA:MIN:0.5:1:600 RRA:MIN:0.5:8:675 RRA:MIN:0.5:96:789 RRA:MAX:0.5:1:600 RRA:MAX:0.5:8:675 RRA:MAX:0.5:96:789 RRA:LAST:0.5:1:600 RRA:LAST:0.5:8:675 RRA:LAST:0.5:96:789 }; print "Creating rrd: $rrd...\n" if $VERBOSE; RRDs::create @params; my $err=RRDs::error; warn "ERROR while creating $rrd: $err\n" if $err; } sub get_last($) { my ($rrd) = @_; my $last = 0; if ( -e $rrd) { $last = RRDs::last($rrd); my $err = RRDs::error; warn "ERROR while getting last for $rrd: $err\n" if $err; }; return $last; }; check_exists_running($RRD, 1); my $last = get_last($RRD); opendir(DIR, $RRD_DIR) || die ("Cannot opendir $RRD_DIR: $!\n"); my @rrdfiles = grep { /\.rrd$/ } readdir (DIR); closedir(DIR); my %last_node; for my $rrdfile (@rrdfiles) { my $nodename = $rrdfile; $nodename =~ s/\.rrd$//; $last_node{$nodename} = get_last($RRD_DIR.'/'.$rrdfile); } my @dirfiles; if (scalar @ARGV) { @dirfiles = @ARGV; } else { opendir(DIR, $DIR_DIR) || die ("Cannot opendir $DIR_DIR: $!\n"); @dirfiles = sort { ($a cmp $b) } grep { /^directory-/ } readdir (DIR); closedir(DIR); } my %KEY_TO_FPR_HASH = (); my %router_hashes; my %hashes_router; my @updateGlobal; my %updates; my %updates_traffic; for my $dir (@dirfiles) { print "Doing $dir...\n" if $VERBOSE; open (DIRECTORY, $DIR_DIR.'/'.$dir) || die ("Cannot open $DIR_DIR/$dir: $!\n"); # published 2004-06-11 02:07:01 # recommended-software 0.0.6.2,0.0.7pre1-cvs-2,0.0.7pre1,0.0.7rc1,0.0.7rc1-cvs,0.0.7rc2,0.0.7 # running-routers moria1 moria2 peacetime metacolo ovmja.... my $dir_published = undef; my $running_routers = undef; my $router_status = undef; my $linenr = 0; while () { $linenr++; chomp; if (/^published (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/) { if (!defined $dir_published) { $dir_published = parsedate($1); } else { warn ("dir_published already set to '$dir_published' in directory $dir, line $linenr\n"); next; }; }; if (/^running-routers\s*(.*?)\s*$/) { if (!defined $running_routers) { $running_routers = $1 } else { warn ("running_routers already set to '$running_routers' in directory $dir, line $linenr\n"); next; }; }; if (/^router-status\s*(.*?)\s*$/) { if (!defined $router_status) { $router_status = $1 } else { warn ("router_status already set to '$router_status' in directory $dir, line $linenr\n"); next; }; }; last if (/^\s*$/); }; die ("Didn't find published for directory") unless defined $dir_published; die ("Didn't find running-routers or router-status") unless (defined $running_routers or $router_status); if (defined $router_status && !defined $running_routers) { $router_status =~ s/=\S+//g; $running_routers = $router_status; }; my $current_router = undef; my $current_platform = undef; my $current_published = undef; my $current_capacity = undef; my $current_hash = undef; my %history; my %router_desc; my $current_write_history = undef; my $current_read_history = undef; my $current_port80 = undef; while () { $linenr++; chomp; if (/^router\s+(\S+)\s/) { if (!defined $current_router) { $current_router = $1; } else { warn ("current_router already set to '$current_router' in directory $dir, line $linenr\n"); next; }; }; if (/^platform\s+(.*)/) { if (!defined $current_platform) { $current_platform = $1; } else { warn ("current_platform already set to '$current_platform' in directory $dir, line $linenr\n"); next; }; }; if (/^published (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/) { if (!defined $current_published) { $current_published = $1; } else { warn ("current_published already set to '$current_published' in directory $dir, line $linenr\n"); next; }; }; if (/^bandwidth (\d+)\s+\d+\s+(\d+)$/) { if (!defined $current_capacity) { my $rate_limit = $1; $current_capacity = $2; $current_capacity = $rate_limit if ($rate_limit < $current_capacity); } else { warn ("current_capacity already set to '$current_capacity' in directory $dir, line $linenr\n"); next; }; }; if (/^opt write-history\s+(.*)/) { if (!defined $current_write_history) { $current_write_history = $1; } else { warn ("current_write_history already set to '$current_write_history' in directory $dir, line $linenr\n"); next; }; }; if (/^opt read-history\s+(.*)/) { if (!defined $current_read_history) { $current_read_history = $1; } else { warn ("current_read_history already set to '$current_read_history' in directory $dir, line $linenr\n"); next; }; }; if (/^(accept|reject) \*:(.*)/ && !defined $current_port80) { my $policy = $1; my $port = $2; if ($port eq '*') { $current_port80 = $policy eq 'accept'; } elsif ($port =~ /^(\d+)$/) { $current_port80 = $policy eq 'accept' if $port == 80; } elsif ($port =~ /^(\d+)-(\d+)$/) { my $min = $1; my $max = $2; $current_port80 = $policy eq 'accept' if $min <= 80 && 80 <= $max; } else { warn ("Cannot parse port spec '$port' in directory $dir, line $linenr\n"); next; } }; if (/^signing-key/) { if (!defined $current_hash) { if ($linenr++ && !~ /^-----BEGIN RSA PUBLIC KEY-----/) { warn ("line after signing key not in expected form\n"); $current_router = undef; $current_platform = undef; $current_published = undef; $current_capacity = undef; $current_port80 = undef; next; }; my $key = ''; my $line; while ($linenr++ && ($line = )) { last if ($line =~ /^-----END RSA PUBLIC KEY-----/); $key .= $line; } if (!defined $line) { warn ("No END RSA PUBLIC KEY in signing key of $current_router ($dir)\n"); last; }; if (defined $KEY_TO_FPR_HASH{$key}) { $current_hash = $KEY_TO_FPR_HASH{$key}; } else { $current_hash = uc(sha1_hex(decode_base64($key))); $KEY_TO_FPR_HASH{$key} = $current_hash; }; } else { warn ("current_hash already set to '$current_hash' in directory $dir, line $linenr\n"); next; }; }; if (/^\s*$/) { if (!defined $current_router) { warn("end of block without current_router\n"); next; }; if (!defined $current_platform) { warn("end of block without current_platform\n"); next; }; if (!defined $current_published) { warn("end of block without current_published\n"); next; }; if (!defined $current_capacity) { if ($current_platform =~ /^Tor 0\.0\.(6|7|8pre1-cvs)/) { $current_capacity = 'U'; } else { warn("end of block without current_capacity in directory $dir, line $linenr\n"); exit; }; }; if (!defined $current_port80) { warn("end of block without current_port80\n"); next; }; if (!defined $current_hash) { warn("end of block without current_hash\n"); next; }; $router_hashes{$current_router} = $current_hash; $hashes_router{$current_hash} = { name => $current_router, platform => $current_platform, published => $current_published }; $router_desc{$current_hash} = { capacity => $current_capacity, port80 => $current_port80 }; $history{$current_hash} = { write_history => $current_write_history, read_history => $current_read_history}; $current_router = undef; $current_platform = undef; $current_published = undef; $current_capacity = undef; $current_port80 = undef; $current_hash = undef; $current_write_history = undef; $current_read_history = undef; } } close (DIRECTORY); # find out which nodes are running my @running_routers = split /\s+/, $running_routers; @running_routers = grep {! /^!/ } @running_routers; my @verified_routers; my @verified_routers_port80; my @unverified_routers; my @fast_verified_routers; my @fast_verified_routers_port80; my @fast_unverified_routers; for my $router (@running_routers) { my $hash; if ($router =~ /^\$/) { $hash = substr($router, 1); } else { $hash = $router_hashes{$router}; if (!defined $hash) { warn ("No hash known for $router\n"); next; }; } if (!defined $router_desc{$hash}->{'port80'}) { warn ("port 80 not defined for $router (this most likely means it was in running-routers but there was no server descriptor)\n"); next; }; if (!defined $router_desc{$hash}->{'capacity'}) { warn ("capacity not defined for $router (this most likely means it was in running-routers but there was no server descriptor)\n"); next; }; my $fast = 0; if ($router_desc{$hash}->{'capacity'} eq 'U' || # in earlier times, all nodes where fast $router_desc{$hash}->{'capacity'} >= $MIN_BANDWIDTH_FOR_FAST) { $fast = 1; }; if ($router =~ /^\$/) { push @unverified_routers, $hash; push @fast_unverified_routers, $hash if $fast; } else { push @verified_routers, $hash; push @verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'}); push @fast_verified_routers, $hash if $fast; push @fast_verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'} && $fast); } if (!defined $last_node{$hash} || $dir_published > $last_node{$hash}) { push @{$updates{$hash}}, $dir_published.':'. ($router =~ /^\$/ ? '0:1' : '1:0').':'. $router_desc{$hash}->{'port80'}.':'. $router_desc{$hash}->{'capacity'}; $last_node{$hash} = $dir_published; } } if ($dir_published > $last) { push @updateGlobal, $dir_published.':'. (scalar @verified_routers).':'. (scalar @unverified_routers).':'. (scalar @verified_routers_port80).':'. (scalar @fast_verified_routers).':'. (scalar @fast_unverified_routers).':'. (scalar @fast_verified_routers_port80); $last = $dir_published }; # update traffic stats for all routers for my $hash (keys %history) { my %hist; for my $inout (qw{write read}) { my $hist = $history{$hash}->{$inout.'_history'}; if (defined $hist) { # 2004-09-26 01:45:45 (900 s) 9619165,12880477,10591411... my ($time, $interval, $values) = $hist =~ /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \((\d+) s\)\s?((?:\d+,)*\d+)?$/; warn ("Could not parse history '$hist'\n"),next unless defined $time; my $stamp = parsedate($time); warn ("Could not parse time in '$hist'\n"),next unless defined $stamp; next unless defined $values; my @values = split /,/, $values; for my $v (@values) { $hist{$stamp}->{$inout} = $v; $stamp -= $interval; }; }; }; for my $stamp (sort {$a <=> $b} keys %hist) { if (!defined $last_node{'TRAF-'.$hash} || $stamp > $last_node{'TRAF-'.$hash}) { push @{$updates_traffic{$hash}}, $stamp.':'. (defined $hist{$stamp}->{'write'} ? $hist{$stamp}->{'write'} : '').':'. (defined $hist{$stamp}->{'read'} ? $hist{$stamp}->{'read'} : ''); $last_node{'TRAF-'.$hash} = $stamp; }; }; }; }; if (scalar @updateGlobal != 0) { check_exists_running($RRD, 1); RRDs::update( $RRD, @updateGlobal ); my $err=RRDs::error; warn "ERROR while updating $RRD: $err\n" if $err; }; for my $router (keys %updates) { next unless exists $hashes_router{$router}; my $rrd = $RRD_DIR.'/'.$router.'.rrd'; check_exists_running($rrd, 0); RRDs::update( $rrd, @{$updates{$router}} ); my $err=RRDs::error; warn "ERROR while updating $rrd: $err\n" if $err; my (($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)) = localtime($last_node{$router}); my $stamp = sprintf("%04d%02d%02d%02d%02d.%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec); print "Setting timestamp of $rrd to $stamp\n" if $VERBOSE; system('touch', '-t', $stamp, $rrd); }; for my $router (keys %updates_traffic) { next unless exists $hashes_router{$router}; my $rrd = $RRD_DIR.'/TRAF-'.$router.'.rrd'; check_exists_traffic($rrd); RRDs::update( $rrd, @{$updates_traffic{$router}} ); my $err=RRDs::error; warn "ERROR while updating $rrd: $err\n" if $err; my (($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)) = localtime($last_node{'TRAF-'.$router}); my $stamp = sprintf("%04d%02d%02d%02d%02d.%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec); print "Setting timestamp of $rrd to $stamp\n" if $VERBOSE; system('touch', '-t', $stamp, $rrd); }; for my $hash (keys %hashes_router) { my $f = $INDEX_DIR.'/'.$hash.".name"; open(I, ">$f") or warn ("Cannot open $f: $!\n"), next; print I $hashes_router{$hash}->{'name'},"\n"; print I $hashes_router{$hash}->{'platform'},"\n"; print I $hashes_router{$hash}->{'published'},"\n"; close I; };