Various changes, including hashed directory structures

2 files changed, 94 insertions, 40 deletions

diff --git a/other/tor/bin/rrd-graph b/other/tor/bin/rrd-graph
index 2940f7c..05a9601 100755
--- a/other/tor/bin/rrd-graph
+++ b/other/tor/bin/rrd-graph
@@ -18,37 +18,51 @@ my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes';
 my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index';
 my $RESULT_DIR = 'result.new';
 my $LONG_TIME_MONTHS = 15;
+my $VERBOSE = 0;
 
 # Get list of RRD files
 opendir(D,$RRD_DIR) or die ("Cannot opendir $RRD_DIR: $!\n");
-my @SOURCES = grep { /\.rrd$/ } readdir(D);
-@SOURCES = grep {
-			my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
-			    $atime,$mtime,$ctime,$blksize,$blocks) = stat($RRD_DIR.'/'.$_);
-			$mtime >= $START_AT;
-		} @SOURCES;
+my @dirs = grep { -d $RRD_DIR.'/'.$_ && $_ =~ /^[A-F0-9]{2}$/ } readdir(D);
 closedir(D);
 
+my @SOURCES;
+for my $dir (@dirs) {
+	opendir(D,$RRD_DIR.'/'.$dir) or die ("Cannot opendir $RRD_DIR/$dir: $!\n");
+	my @sources = grep { /\.rrd$/ } readdir(D);
+	@sources = grep { !/^TRAF-/ } @sources;
+	@sources = grep {
+				my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
+				    $atime,$mtime,$ctime,$blksize,$blocks) = stat($RRD_DIR.'/'.$dir.'/'.$_);
+				$mtime >= $START_AT;
+			} @sources;
+	push @SOURCES, @sources;
+	closedir(D);
+};
+
 # Read hash <-> name mapping
 my %hashes_router;
 opendir(D,$INDEX_DIR) or die ("Cannot opendir $INDEX_DIR: $!\n");
-my @NAME = grep { /\.name$/ } readdir(D);
+@dirs = grep { -d $INDEX_DIR.'/'.$_ && $_ =~ /^[A-F0-9]{2}$/ } readdir(D);
 closedir(D);
-for my $hash (@NAME) {
-	my $f = $INDEX_DIR.'/'.$hash;
-	open (F, $f) or warn("Cannot open $f: $!\n"), next;
-	my $name = <F>;
-	my $platform = <F>;
-	my $published = <F>;
-	chomp $name;
-	chomp $platform;
-	chomp $published;
-	close F;
-	$hash =~ s/\.name$//;
-	$hashes_router{$hash} = {name => $name, platform => $platform, published => $published };
+for my $dir (@dirs) {
+	opendir(D,$INDEX_DIR.'/'.$dir) or die ("Cannot opendir $INDEX_DIR/$dir: $!\n");
+	my @NAME = grep { /\.name$/ } readdir(D);
+	closedir(D);
+	for my $hash (@NAME) {
+		my $f = $INDEX_DIR.'/'.$dir.'/'.$hash;
+		open (F, $f) or warn("Cannot open $f: $!\n"), next;
+		my $name = <F>;
+		my $platform = <F>;
+		my $published = <F>;
+		chomp $name;
+		chomp $platform;
+		chomp $published;
+		close F;
+		$hash =~ s/\.name$//;
+		$hashes_router{$hash} = {name => $name, platform => $platform, published => $published };
+	};
 };
 
-@SOURCES = grep { !/^TRAF-/ } @SOURCES;
 @SOURCES = sort {
 		my $aa = $a;
 		my $bb = $b;
@@ -69,21 +83,24 @@ for my $source (@SOURCES) {
 	my $hash = $source;
 	   $hash =~ s/\.rrd$//;
 	my $name = $hashes_router{$hash}->{'name'};
+	die ("No name for $hash") unless defined $name;
 	my $platform = $hashes_router{$hash}->{'platform'};
 	my $published = $hashes_router{$hash}->{'published'};
 	my $def = $name;
 	my $cdef = 'c_'.$def;
 	my $label = $def;
+	my $subdir = substr($hash,0,2);
 
-	my $source_traffic = $RRD_DIR .'/TRAF-'. $source;
-	$source = $RRD_DIR .'/'. $source;
+	my $source_traffic = $RRD_DIR .'/'.$subdir.'/TRAF-'. $source;
+	$source = $RRD_DIR .'/'.$subdir.'/'. $source;
 
 	my (undef,undef,undef,undef,undef,undef,undef,undef, undef,$mtime,undef,undef,undef) = stat($source);
 	my $section = $mtime < $OLD_AT ? 'old' : 'current';
 
 
 	my @this;
-	push @this, sprintf($RESULT_DIR.'/node-%s.png', $hash);
+	mkdir ($RESULT_DIR.'/'.$subdir) unless -d ($RESULT_DIR.'/'.$subdir);
+	push @this, sprintf($RESULT_DIR.'/'.$subdir.'/node-%s.png', $hash);
 	push @this, sprintf('-s %s', $START_AT);
 	push @this, sprintf('-e %s', $END_AT);
 	push @this, sprintf('-t %s', "$name");
@@ -108,6 +125,7 @@ for my $source (@SOURCES) {
 	push @this, sprintf('CDEF:%s=d_runningVerified,0,*,%s,+'     , 'zeroline'         , '0');
 	push @this, sprintf('%s:%s' , 'LINE1' , 'zeroline' );
 
+	print "Graphing $hash running.\n" if $VERBOSE;
 	my ($averages,$xsize,$ysize) = RRDs::graph @this;
 	warn RRDs::error if RRDs::error;
 
@@ -118,7 +136,7 @@ for my $source (@SOURCES) {
 		"platform $platform<br>".
 		"published $published".
 		"</blockquote>".
-		"<p><img width=\"$xsize\" height=\"$ysize\" alt=\"$name running\" src=\"$IMG_URL"."node-$hash.png\">\n";
+		"<p><img width=\"$xsize\" height=\"$ysize\" alt=\"$name running\" src=\"$IMG_URL"."$subdir/node-$hash.png\">\n";
 	$index{$section} .= "<li><a href=\"$section.html#$hash\">$name</a></li>\n";
 
 	if (-e $source_traffic) {
@@ -159,7 +177,7 @@ for my $source (@SOURCES) {
 
 		for my $thumb (0, 1) {
 			@this = ();
-			push @this, sprintf($RESULT_DIR.'/node-'.($thumb ? 'small-' : '').'traf-%s.png', $hash);
+			push @this, sprintf($RESULT_DIR.'/'.$subdir.'/node-'.($thumb ? 'small-' : '').'traf-%s.png', $hash);
 			#push @this, sprintf('-s %s', '-604800');
 			push @this, sprintf('-s %s', $START_AT);
 			push @this, sprintf('-e %s', $END_AT);
@@ -204,12 +222,13 @@ for my $source (@SOURCES) {
 			#push @this, sprintf('CDEF:%s=read,0,*,%s,+'     , 'zeroline'         , '0');
 			#push @this, sprintf('%s:%s' , 'LINE1' , 'zeroline' );
 
+			print "Graphing $hash traffic $thumb.\n" if $VERBOSE;
 			($averages,$xsize,$ysize) = RRDs::graph @this;
 			warn RRDs::error if RRDs::error;
 
 		};
-		$body{$section} .= " <a href=\"$IMG_URL"."node-traf-$hash.png\">";
-		$body{$section} .= "<img width=\"$xsize\" height=\"$ysize\" alt=\"$name traffic\" src=\"$IMG_URL"."node-small-traf-$hash.png\">";
+		$body{$section} .= " <a href=\"$IMG_URL"."$subdir/node-traf-$hash.png\">";
+		$body{$section} .= "<img width=\"$xsize\" height=\"$ysize\" alt=\"$name traffic\" src=\"$IMG_URL"."$subdir/node-small-traf-$hash.png\">";
 		$body{$section} .= "</a>";
 	};
 	#$body{$section} .= "<br><a href=\"#_top\">top</a>";
@@ -253,6 +272,7 @@ sub makeTotalTrafficPic($$$) {
 	push @totalTraffic, sprintf('CDEF:zeroline=aggread%d,0,*', $traf_item_counter);
 	push @totalTraffic, sprintf('%s:%s' , 'LINE1' , 'zeroline' );
 
+	print "Graphing total traffic $file $age $include15min.\n" if $VERBOSE;
 	my ($averages,$xsize,$ysize) = RRDs::graph @totalTraffic;
 	warn RRDs::error if RRDs::error;
 
@@ -307,6 +327,7 @@ sub makeNumbersGraph($$) {
 
 
 
+	print "Graphing numbers $file $age.\n" if $VERBOSE;
 	my ($averages,$xsize,$ysize) = RRDs::graph @params;
 	warn RRDs::error if RRDs::error;
 
diff --git a/other/tor/bin/rrd-update b/other/tor/bin/rrd-update
index c7062f7..30e5466 100755
--- a/other/tor/bin/rrd-update
+++ b/other/tor/bin/rrd-update
@@ -7,7 +7,7 @@ use Digest::SHA1 qw(sha1_hex);
 use Time::ParseDate;
 
 my $NOW = time;
-my $VERBOSE = 1;
+my $VERBOSE = 0;
 my $RRD = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/running_Routers.rrd';
 my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes';
 my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index';
@@ -85,15 +85,32 @@ sub get_last($) {
 check_exists_running($RRD, 1);
 my $last = get_last($RRD);
 
-opendir(DIR, $RRD_DIR) || die ("Cannot opendir $RRD_DIR: $!\n");
-my @rrdfiles = grep { /\.rrd$/ } readdir (DIR);
-closedir(DIR);
 my %last_node;
-for my $rrdfile (@rrdfiles) {
-	my $nodename = $rrdfile;
-	$nodename =~ s/\.rrd$//;
-	$last_node{$nodename} = get_last($RRD_DIR.'/'.$rrdfile);
-}
+sub fetch_last_node($) {
+	my ($hash) = @_;
+
+	my ($subdir) = $hash =~ /^([A-F0-9]{2})/;
+	die "Unknown subdir for router $hash" unless defined $subdir;
+	return unless -d $RRD_DIR.'/'.$subdir;
+	my $rrd = $RRD_DIR.'/'.$subdir.'/'.$hash.'.rrd';
+	$last_node{$hash} = get_last($rrd);
+	$rrd = $RRD_DIR.'/'.$subdir.'/TRAF-'.$hash.'.rrd';
+	$last_node{'TRAF-'.$hash} = get_last($rrd);
+};
+#opendir(DIR, $RRD_DIR) || die ("Cannot opendir $RRD_DIR: $!\n");
+#my @subdirs = grep { -d $RRD_DIR.'/'.$_ && $_ =~ /^[A-F0-9]{2}$/} readdir (DIR);
+#closedir(DIR);
+
+#for my $dir (@subdirs) {
+#	opendir(DIR, $RRD_DIR.'/'.$dir) || die ("Cannot opendir $RRD_DIR/$dir: $!\n");
+#	my @rrdfiles = grep { /\.rrd$/ } readdir (DIR);
+#	closedir(DIR);
+#	for my $rrdfile (@rrdfiles) {
+#		my $nodename = $rrdfile;
+#		$nodename =~ s/\.rrd$//;
+#		$last_node{$nodename} = get_last($RRD_DIR.'/'.$dir.'/'.$rrdfile);
+#	};
+#};
 
 my @dirfiles;
 if (scalar @ARGV) {
@@ -169,9 +186,12 @@ for my $dir (@dirfiles) {
 	my $current_write_history = undef;
 	my $current_read_history = undef;
 	my $current_port80 = undef;
+	my $last_was_empty = 0;
 	while (<DIRECTORY>) {
 		$linenr++;
 		chomp;
+		next if ($last_was_empty == 1 && /^\s*$/);
+		$last_was_empty = 0;
 		if (/^router\s+(\S+)\s/) {
 			if (!defined $current_router) {
 				$current_router = $1;
@@ -315,6 +335,8 @@ for my $dir (@dirfiles) {
 			$current_hash = undef;
 			$current_write_history = undef;
 			$current_read_history = undef;
+
+			$last_was_empty = 1
 		}
 	}
 	close (DIRECTORY);
@@ -367,6 +389,7 @@ for my $dir (@dirfiles) {
 			push @fast_verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'} && $fast);
 		}
 
+		fetch_last_node($hash) unless (defined $last_node{$hash});
 		if (!defined $last_node{$hash} || $dir_published > $last_node{$hash}) {
 			push @{$updates{$hash}}, $dir_published.':'.
 				($router =~ /^\$/ ? '0:1' : '1:0').':'.
@@ -394,7 +417,7 @@ for my $dir (@dirfiles) {
 			if (defined $hist) {
 				# 2004-09-26 01:45:45 (900 s) 9619165,12880477,10591411...
 				my ($time, $interval, $values) =
-					$hist =~ /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \((\d+) s\)\s?((?:\d+,)*\d+)?$/;
+					$hist =~ /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \((\d+) s\)\s?((?:-?\d+,)*-?\d+)?$/;
 				warn ("Could not parse history '$hist'\n"),next
 					unless defined $time;
 				my $stamp = parsedate($time);
@@ -403,11 +426,15 @@ for my $dir (@dirfiles) {
 				next unless defined $values;
 				my @values = split /,/, $values;
 				for my $v (@values) {
+					if ($v < 0) {
+						$v += 4294967296;  # work around bug in tor, that has history as signed 32 bit integer.
+					};
 					$hist{$stamp}->{$inout} = $v;
 					$stamp -= $interval;
 				};
 			};
 		};
+		fetch_last_node($hash) unless (defined $last_node{'TRAF-'.$hash});
 		for my $stamp (sort {$a <=> $b} keys %hist) {
 			if (!defined $last_node{'TRAF-'.$hash} || $stamp > $last_node{'TRAF-'.$hash}) {
 				push @{$updates_traffic{$hash}}, $stamp.':'.
@@ -428,7 +455,9 @@ if (scalar @updateGlobal != 0) {
 
 for my $router (keys %updates) {
 	next unless exists $hashes_router{$router};
-	my $rrd = $RRD_DIR.'/'.$router.'.rrd';
+	my ($subdir) = substr($router,0,2);
+	mkdir($RRD_DIR.'/'.$subdir) unless -d $RRD_DIR.'/'.$subdir;
+	my $rrd = $RRD_DIR.'/'.$subdir.'/'.$router.'.rrd';
 	check_exists_running($rrd, 0);
 	RRDs::update( $rrd, @{$updates{$router}} );
 	my $err=RRDs::error;
@@ -441,7 +470,9 @@ for my $router (keys %updates) {
 
 for my $router (keys %updates_traffic) {
 	next unless exists $hashes_router{$router};
-	my $rrd = $RRD_DIR.'/TRAF-'.$router.'.rrd';
+	my ($subdir) = substr($router,0,2);
+	mkdir($RRD_DIR.'/'.$subdir) unless -d $RRD_DIR.'/'.$subdir;
+	my $rrd = $RRD_DIR.'/'.$subdir.'/TRAF-'.$router.'.rrd';
 	check_exists_traffic($rrd);
 	RRDs::update( $rrd, @{$updates_traffic{$router}} );
 	my $err=RRDs::error;
@@ -455,7 +486,9 @@ for my $router (keys %updates_traffic) {
 
 
 for my $hash (keys %hashes_router) {
-	my $f = $INDEX_DIR.'/'.$hash.".name";
+	my ($subdir) = substr($hash,0,2);
+	mkdir($INDEX_DIR.'/'.$subdir) unless -d $INDEX_DIR.'/'.$subdir;
+	my $f = $INDEX_DIR.'/'.$subdir.'/'.$hash.".name";
 	open(I, ">$f") or warn ("Cannot open $f: $!\n"), next;
 	print I $hashes_router{$hash}->{'name'},"\n";
 	print I $hashes_router{$hash}->{'platform'},"\n";