#!/usr/bin/perl -w

use strict;
use RRDs;
use MIME::Base64;
use Digest::SHA1 qw(sha1_hex);
use Time::ParseDate;

my $NOW = time;
my $VERBOSE = 0;
my $RRD = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/running_Routers.rrd';
my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes';
my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index';
my $DIR_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/tor-directory';
my $MIN_BANDWIDTH_FOR_FAST = 20000;

sub check_exists_running($$) {
	my ($rrd, $global) = @_;
	return if (-e $rrd);
	my @params = ($rrd);
	push @params,  '-b', 'now - 24 months',
	 qw{ --step 3600
	  DS:runningVerified:GAUGE:172800:U:U
	  DS:runningUnverified:GAUGE:172800:U:U
	  DS:exit80Verified:GAUGE:172800:U:U };
	if ($global) {
		push @params, qw{
		  DS:fastRunningVerified:GAUGE:172800:U:U
		  DS:fastRunningUnverifi:GAUGE:172800:U:U
		  DS:fastExit80Verified:GAUGE:172800:U:U };
	} else {
		push @params, 'DS:capacity:GAUGE:172800:U:U';
	};
	push @params, qw{
		RRA:MIN:0.5:1:336
		RRA:MAX:0.5:1:336
		RRA:AVERAGE:0.5:1:336
		RRA:LAST:0.5:1:336
		RRA:MIN:0.5:24:730
		RRA:MAX:0.5:24:730
		RRA:AVERAGE:0.5:24:730
		};
	print "Creating rrd: $rrd...\n" if $VERBOSE;
	RRDs::create @params;
	my $err=RRDs::error;
	warn "ERROR while creating $rrd: $err\n" if $err;
}
sub check_exists_traffic($) {
	my ($rrd) = @_;
	return if (-e $rrd);
	my @params = ($rrd);
	push @params,  '-b', 'now - 12 months',
	 qw{ --step 900
		DS:write:ABSOLUTE:900:U:U
		DS:read:ABSOLUTE:900:U:U
		RRA:AVERAGE:0.5:1:600
		RRA:AVERAGE:0.5:8:675
		RRA:AVERAGE:0.5:96:789
		RRA:MIN:0.5:1:600
		RRA:MIN:0.5:8:675
		RRA:MIN:0.5:96:789
		RRA:MAX:0.5:1:600
		RRA:MAX:0.5:8:675
		RRA:MAX:0.5:96:789
		RRA:LAST:0.5:1:600
		RRA:LAST:0.5:8:675
		RRA:LAST:0.5:96:789
	};
	print "Creating rrd: $rrd...\n" if $VERBOSE;
	RRDs::create @params;
	my $err=RRDs::error;
	warn "ERROR while creating $rrd: $err\n" if $err;
}
sub get_last($) {
	my ($rrd) = @_;
	my $last = 0;
	if ( -e $rrd) {
		$last = RRDs::last($rrd);
		my $err = RRDs::error;
		warn "ERROR while getting last for $rrd: $err\n" if $err;
	};
	return $last;
};

check_exists_running($RRD, 1);
my $last = get_last($RRD);

my %last_node;
sub fetch_last_node($) {
	my ($hash) = @_;

	my ($subdir) = $hash =~ /^([A-F0-9]{2})/;
	die "Unknown subdir for router $hash" unless defined $subdir;
	return unless -d $RRD_DIR.'/'.$subdir;
	my $rrd = $RRD_DIR.'/'.$subdir.'/'.$hash.'.rrd';
	$last_node{$hash} = get_last($rrd);
	$rrd = $RRD_DIR.'/'.$subdir.'/TRAF-'.$hash.'.rrd';
	$last_node{'TRAF-'.$hash} = get_last($rrd);
};
#opendir(DIR, $RRD_DIR) || die ("Cannot opendir $RRD_DIR: $!\n");
#my @subdirs = grep { -d $RRD_DIR.'/'.$_ && $_ =~ /^[A-F0-9]{2}$/} readdir (DIR);
#closedir(DIR);

#for my $dir (@subdirs) {
#	opendir(DIR, $RRD_DIR.'/'.$dir) || die ("Cannot opendir $RRD_DIR/$dir: $!\n");
#	my @rrdfiles = grep { /\.rrd$/ } readdir (DIR);
#	closedir(DIR);
#	for my $rrdfile (@rrdfiles) {
#		my $nodename = $rrdfile;
#		$nodename =~ s/\.rrd$//;
#		$last_node{$nodename} = get_last($RRD_DIR.'/'.$dir.'/'.$rrdfile);
#	};
#};

my @dirfiles;
if (scalar @ARGV) {
	@dirfiles = @ARGV;
} else {
	opendir(DIR, $DIR_DIR) || die ("Cannot opendir $DIR_DIR: $!\n");
	@dirfiles = sort { ($a cmp $b) } grep { /^directory-/ } readdir (DIR);
	closedir(DIR);
}

my %KEY_TO_FPR_HASH = ();

my %router_hashes;
my %hashes_router;
my @updateGlobal;
my %updates;
my %updates_traffic;
for my $dir (@dirfiles) {
	print "Doing $dir...\n" if $VERBOSE;
	open (DIRECTORY, $DIR_DIR.'/'.$dir) || die ("Cannot open $DIR_DIR/$dir: $!\n");
	# published 2004-06-11 02:07:01
	# recommended-software 0.0.6.2,0.0.7pre1-cvs-2,0.0.7pre1,0.0.7rc1,0.0.7rc1-cvs,0.0.7rc2,0.0.7
	# running-routers moria1 moria2 peacetime metacolo ovmja....
	my $dir_published = undef;
	my $running_routers = undef;
	my $router_status = undef;
	my $linenr = 0;
	while (<DIRECTORY>) {
		$linenr++;
		chomp;
		if (/^published (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/) {
			if (!defined $dir_published) {
				$dir_published = parsedate($1);
			} else {
				warn ("dir_published already set to '$dir_published' in directory $dir, line $linenr\n");
				next;
			};
		};
		if (/^running-routers\s*(.*?)\s*$/) {
			if (!defined $running_routers) {
				$running_routers = $1
			} else {
				warn ("running_routers already set to '$running_routers' in directory $dir, line $linenr\n");
				next;
			};
		};
		if (/^router-status\s*(.*?)\s*$/) {
			if (!defined $router_status) {
				$router_status = $1
			} else {
				warn ("router_status already set to '$router_status' in directory $dir, line $linenr\n");
				next;
			};
		};
		last if (/^\s*$/);
	};
	die ("Didn't find published for directory") unless defined $dir_published;
	die ("Didn't find running-routers or router-status") unless (defined $running_routers or $router_status);

	if (defined $router_status && !defined $running_routers) {
		$router_status =~ s/=\S+//g;
		$running_routers = $router_status;
	};


	my $current_router = undef;
	my $current_platform = undef;
	my $current_published = undef;
	my $current_capacity = undef;
	my $current_hash = undef;
	my %history;
	my %router_desc;
	my $current_write_history = undef;
	my $current_read_history = undef;
	my $current_port80 = undef;
	my $last_was_empty = 0;
	while (<DIRECTORY>) {
		$linenr++;
		chomp;
		next if ($last_was_empty == 1 && /^\s*$/);
		$last_was_empty = 0;
		if (/^router\s+(\S+)\s/) {
			if (!defined $current_router) {
				$current_router = $1;
			} else {
				warn ("current_router already set to '$current_router' in directory $dir, line $linenr\n");
				next;
			};
		};
		if (/^platform\s+(.*)/) {
			if (!defined $current_platform) {
				$current_platform = $1;
			} else {
				warn ("current_platform already set to '$current_platform' in directory $dir, line $linenr\n");
				next;
			};
		};
		if (/^published (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})$/) {
			if (!defined $current_published) {
				$current_published = $1;
			} else {
				warn ("current_published already set to '$current_published' in directory $dir, line $linenr\n");
				next;
			};
		};
		if (/^bandwidth (\d+)\s+\d+\s+(\d+)$/) {
			if (!defined $current_capacity) {
				my $rate_limit = $1;
				$current_capacity = $2;
				$current_capacity = $rate_limit if ($rate_limit < $current_capacity);
			} else {
				warn ("current_capacity already set to '$current_capacity' in directory $dir, line $linenr\n");
				next;
			};
		};
		if (/^opt write-history\s+(.*)/) {
			if (!defined $current_write_history) {
				$current_write_history = $1;
			} else {
				warn ("current_write_history already set to '$current_write_history' in directory $dir, line $linenr\n");
				next;
			};
		};
		if (/^opt read-history\s+(.*)/) {
			if (!defined $current_read_history) {
				$current_read_history = $1;
			} else {
				warn ("current_read_history already set to '$current_read_history' in directory $dir, line $linenr\n");
				next;
			};
		};
		if (/^(accept|reject) \*:(.*)/ && !defined $current_port80) {
			my $policy = $1;
			my $port = $2;
			if ($port eq '*') {
				$current_port80 = $policy eq 'accept';
			} elsif ($port =~ /^(\d+)$/) {
				$current_port80 = $policy eq 'accept' if $port == 80;
			} elsif ($port =~ /^(\d+)-(\d+)$/) {
				my $min = $1;
				my $max = $2;
				$current_port80 = $policy eq 'accept' if $min <= 80 && 80 <= $max;
			} else {
				warn ("Cannot parse port spec '$port' in directory $dir, line $linenr\n");
				next;
			}
		};
		if (/^signing-key/) {
			if (!defined $current_hash) {
				if ($linenr++ && <DIRECTORY> !~ /^-----BEGIN RSA PUBLIC KEY-----/) {
					warn ("line after signing key not in expected form\n");
					$current_router = undef;
					$current_platform = undef;
					$current_published = undef;
					$current_capacity = undef;
					$current_port80 = undef;
					next;
				};
				my $key = '';
				my $line;
				while ($linenr++ && ($line = <DIRECTORY>)) {
					last if ($line =~ /^-----END RSA PUBLIC KEY-----/);
					$key .= $line;
				}
				if (!defined $line) {
					warn ("No END RSA PUBLIC KEY in signing key of $current_router ($dir)\n");
					last;
				};
				if (defined $KEY_TO_FPR_HASH{$key}) {
					$current_hash = $KEY_TO_FPR_HASH{$key};
				} else {
					$current_hash = uc(sha1_hex(decode_base64($key)));
					$KEY_TO_FPR_HASH{$key} = $current_hash;
				};
			} else {
				warn ("current_hash already set to '$current_hash' in directory $dir, line $linenr\n");
				next;
			};
		};
		if (/^\s*$/) {
			if (!defined $current_router) {
				warn("end of block without current_router\n");
				next;
			};
			if (!defined $current_platform) {
				warn("end of block without current_platform\n");
				next;
			};
			if (!defined $current_published) {
				warn("end of block without current_published\n");
				next;
			};
			if (!defined $current_capacity) {
				if ($current_platform =~ /^Tor 0\.0\.(6|7|8pre1-cvs)/) {
					$current_capacity = 'U';
				} else {
					warn("end of block without current_capacity in directory $dir, line $linenr\n");
					exit;
				};
			};
			if (!defined $current_port80) {
				warn("end of block without current_port80\n");
				next;
			};
			if (!defined $current_hash) {
				warn("end of block without current_hash\n");
				next;
			};
			$router_hashes{$current_router} = $current_hash;
			$hashes_router{$current_hash} = { name => $current_router,
			                                  platform => $current_platform,
			                                  published => $current_published };
			$router_desc{$current_hash} = { capacity => $current_capacity,
			                                port80 => $current_port80 };
			$history{$current_hash} = { write_history => $current_write_history,
			                            read_history => $current_read_history};
			$current_router = undef;
			$current_platform = undef;
			$current_published = undef;
			$current_capacity = undef;
			$current_port80 = undef;
			$current_hash = undef;
			$current_write_history = undef;
			$current_read_history = undef;

			$last_was_empty = 1
		}
	}
	close (DIRECTORY);

	# find out which nodes are running
	my @running_routers = split /\s+/, $running_routers;
	@running_routers = grep {! /^!/ }  @running_routers;
	my @verified_routers;
	my @verified_routers_port80;
	my @unverified_routers;
	my @fast_verified_routers;
	my @fast_verified_routers_port80;
	my @fast_unverified_routers;
	for my $router (@running_routers) {
		my $hash;
		if ($router =~ /^\$/) {
			$hash = substr($router, 1);
		} else {
			$hash = $router_hashes{$router};
			if (!defined $hash) {
				warn ("No hash known for $router\n");
				next;
			};
		}

		if (!defined $router_desc{$hash}->{'port80'}) {
			warn ("port 80 not defined for $router (this most likely means it was in running-routers but there was no server descriptor)\n");
			next;
		};
		if (!defined $router_desc{$hash}->{'capacity'}) {
			warn ("capacity not defined for $router (this most likely means it was in running-routers but there was no server descriptor)\n");
			next;
		};

		my $fast = 0;
		if ($router_desc{$hash}->{'capacity'} eq 'U' ||  # in earlier times, all nodes where fast
			$router_desc{$hash}->{'capacity'} >= $MIN_BANDWIDTH_FOR_FAST) {
			$fast = 1;
		};


		if ($router =~ /^\$/) {
			push @unverified_routers, $hash;
			push @fast_unverified_routers, $hash if $fast;
		} else {
			push @verified_routers, $hash;
			push @verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'});

			push @fast_verified_routers, $hash if $fast;
			push @fast_verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'} && $fast);
		}

		fetch_last_node($hash) unless (defined $last_node{$hash});
		if (!defined $last_node{$hash} || $dir_published > $last_node{$hash}) {
			push @{$updates{$hash}}, $dir_published.':'.
				($router =~ /^\$/ ? '0:1' : '1:0').':'.
				$router_desc{$hash}->{'port80'}.':'.
				$router_desc{$hash}->{'capacity'};
			$last_node{$hash} = $dir_published;
		}
	}
	if ($dir_published > $last) {
		push @updateGlobal, $dir_published.':'.
			(scalar @verified_routers).':'.
			(scalar @unverified_routers).':'.
			(scalar @verified_routers_port80).':'.
			(scalar @fast_verified_routers).':'.
			(scalar @fast_unverified_routers).':'.
			(scalar @fast_verified_routers_port80);
		$last = $dir_published
	};

	# update traffic stats for all routers
	for my $hash (keys %history) {
		my %hist;
		for my $inout (qw{write read}) {
			my $hist =  $history{$hash}->{$inout.'_history'};
			if (defined $hist) {
				# 2004-09-26 01:45:45 (900 s) 9619165,12880477,10591411...
				my ($time, $interval, $values) =
					$hist =~ /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \((\d+) s\)\s?((?:-?\d+,)*-?\d+)?$/;
				warn ("Could not parse history '$hist'\n"),next
					unless defined $time;
				my $stamp = parsedate($time);
				warn ("Could not parse time in '$hist'\n"),next
					unless defined $stamp;
				next unless defined $values;
				my @values = split /,/, $values;
				for my $v (@values) {
					if ($v < 0) {
						$v += 4294967296;  # work around bug in tor, that has history as signed 32 bit integer.
					};
					if ($v > 18446744000000000000) {
						$v = $v - 2**64 + 2**32; # state loading bug, fixed 2006-01-02
					};
					#print "Setting $inout value to $v for host $hash from $dir\n";
					$hist{$stamp}->{$inout} = $v;
					$stamp -= $interval;
				};
			};
		};
		fetch_last_node($hash) unless (defined $last_node{'TRAF-'.$hash});
		for my $stamp (sort {$a <=> $b} keys %hist) {
			if (!defined $last_node{'TRAF-'.$hash} || $stamp > $last_node{'TRAF-'.$hash}) {
				push @{$updates_traffic{$hash}}, $stamp.':'.
					(defined $hist{$stamp}->{'write'} ? $hist{$stamp}->{'write'} : '').':'.
					(defined $hist{$stamp}->{'read'} ? $hist{$stamp}->{'read'} : '');
				$last_node{'TRAF-'.$hash} = $stamp;
			};
		};
	};
};

if (scalar @updateGlobal != 0) {
	check_exists_running($RRD, 1);
	RRDs::update( $RRD, @updateGlobal );
	my $err=RRDs::error;
	warn "ERROR while updating $RRD: $err\n" if $err;
};

for my $router (keys %updates) {
	next unless exists $hashes_router{$router};
	my ($subdir) = substr($router,0,2);
	mkdir($RRD_DIR.'/'.$subdir) unless -d $RRD_DIR.'/'.$subdir;
	my $rrd = $RRD_DIR.'/'.$subdir.'/'.$router.'.rrd';
	check_exists_running($rrd, 0);
	RRDs::update( $rrd, @{$updates{$router}} );
	my $err=RRDs::error;
	warn "ERROR while updating $rrd: $err\n" if $err;
	my (($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)) = localtime($last_node{$router});
	my $stamp = sprintf("%04d%02d%02d%02d%02d.%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
	print "Setting timestamp of $rrd to $stamp\n" if $VERBOSE;
	system('touch', '-t', $stamp, $rrd);
};

for my $router (keys %updates_traffic) {
	next unless exists $hashes_router{$router};
	my ($subdir) = substr($router,0,2);
	mkdir($RRD_DIR.'/'.$subdir) unless -d $RRD_DIR.'/'.$subdir;
	my $rrd = $RRD_DIR.'/'.$subdir.'/TRAF-'.$router.'.rrd';
	check_exists_traffic($rrd);
	RRDs::update( $rrd, @{$updates_traffic{$router}} );
	my $err=RRDs::error;
	warn "ERROR while updating $rrd: $err\n" if $err;
	my (($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)) = localtime($last_node{'TRAF-'.$router});
	my $stamp = sprintf("%04d%02d%02d%02d%02d.%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
	print "Setting timestamp of $rrd to $stamp\n" if $VERBOSE;
	system('touch', '-t', $stamp, $rrd);
};



for my $hash (keys %hashes_router) {
	my ($subdir) = substr($hash,0,2);
	mkdir($INDEX_DIR.'/'.$subdir) unless -d $INDEX_DIR.'/'.$subdir;
	my $f = $INDEX_DIR.'/'.$subdir.'/'.$hash.".name";
	open(I, ">$f") or warn ("Cannot open $f: $!\n"), next;
	print I $hashes_router{$hash}->{'name'},"\n";
	print I $hashes_router{$hash}->{'platform'},"\n";
	print I $hashes_router{$hash}->{'published'},"\n";
	close I;
};