From c3728a76bd0081cf5a58bad0ed2424946e77f1ec Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Tue, 15 Nov 2005 09:52:14 +0000 Subject: Various changes, including hashed directory structures --- other/tor/bin/rrd-graph | 75 +++++++++++++++++++++++++++++++----------------- other/tor/bin/rrd-update | 59 ++++++++++++++++++++++++++++--------- 2 files changed, 94 insertions(+), 40 deletions(-) diff --git a/other/tor/bin/rrd-graph b/other/tor/bin/rrd-graph index 2940f7c..05a9601 100755 --- a/other/tor/bin/rrd-graph +++ b/other/tor/bin/rrd-graph @@ -18,37 +18,51 @@ my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes'; my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index'; my $RESULT_DIR = 'result.new'; my $LONG_TIME_MONTHS = 15; +my $VERBOSE = 0; # Get list of RRD files opendir(D,$RRD_DIR) or die ("Cannot opendir $RRD_DIR: $!\n"); -my @SOURCES = grep { /\.rrd$/ } readdir(D); -@SOURCES = grep { - my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, - $atime,$mtime,$ctime,$blksize,$blocks) = stat($RRD_DIR.'/'.$_); - $mtime >= $START_AT; - } @SOURCES; +my @dirs = grep { -d $RRD_DIR.'/'.$_ && $_ =~ /^[A-F0-9]{2}$/ } readdir(D); closedir(D); +my @SOURCES; +for my $dir (@dirs) { + opendir(D,$RRD_DIR.'/'.$dir) or die ("Cannot opendir $RRD_DIR/$dir: $!\n"); + my @sources = grep { /\.rrd$/ } readdir(D); + @sources = grep { !/^TRAF-/ } @sources; + @sources = grep { + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, + $atime,$mtime,$ctime,$blksize,$blocks) = stat($RRD_DIR.'/'.$dir.'/'.$_); + $mtime >= $START_AT; + } @sources; + push @SOURCES, @sources; + closedir(D); +}; + # Read hash <-> name mapping my %hashes_router; opendir(D,$INDEX_DIR) or die ("Cannot opendir $INDEX_DIR: $!\n"); -my @NAME = grep { /\.name$/ } readdir(D); +@dirs = grep { -d $INDEX_DIR.'/'.$_ && $_ =~ /^[A-F0-9]{2}$/ } readdir(D); closedir(D); -for my $hash (@NAME) { - my $f = $INDEX_DIR.'/'.$hash; - open (F, $f) or warn("Cannot open $f: $!\n"), next; - my $name = ; - my $platform = ; - my $published = ; - chomp $name; - chomp $platform; - chomp $published; - close F; - $hash =~ s/\.name$//; - $hashes_router{$hash} = {name => $name, platform => $platform, published => $published }; +for my $dir (@dirs) { + opendir(D,$INDEX_DIR.'/'.$dir) or die ("Cannot opendir $INDEX_DIR/$dir: $!\n"); + my @NAME = grep { /\.name$/ } readdir(D); + closedir(D); + for my $hash (@NAME) { + my $f = $INDEX_DIR.'/'.$dir.'/'.$hash; + open (F, $f) or warn("Cannot open $f: $!\n"), next; + my $name = ; + my $platform = ; + my $published = ; + chomp $name; + chomp $platform; + chomp $published; + close F; + $hash =~ s/\.name$//; + $hashes_router{$hash} = {name => $name, platform => $platform, published => $published }; + }; }; -@SOURCES = grep { !/^TRAF-/ } @SOURCES; @SOURCES = sort { my $aa = $a; my $bb = $b; @@ -69,21 +83,24 @@ for my $source (@SOURCES) { my $hash = $source; $hash =~ s/\.rrd$//; my $name = $hashes_router{$hash}->{'name'}; + die ("No name for $hash") unless defined $name; my $platform = $hashes_router{$hash}->{'platform'}; my $published = $hashes_router{$hash}->{'published'}; my $def = $name; my $cdef = 'c_'.$def; my $label = $def; + my $subdir = substr($hash,0,2); - my $source_traffic = $RRD_DIR .'/TRAF-'. $source; - $source = $RRD_DIR .'/'. $source; + my $source_traffic = $RRD_DIR .'/'.$subdir.'/TRAF-'. $source; + $source = $RRD_DIR .'/'.$subdir.'/'. $source; my (undef,undef,undef,undef,undef,undef,undef,undef, undef,$mtime,undef,undef,undef) = stat($source); my $section = $mtime < $OLD_AT ? 'old' : 'current'; my @this; - push @this, sprintf($RESULT_DIR.'/node-%s.png', $hash); + mkdir ($RESULT_DIR.'/'.$subdir) unless -d ($RESULT_DIR.'/'.$subdir); + push @this, sprintf($RESULT_DIR.'/'.$subdir.'/node-%s.png', $hash); push @this, sprintf('-s %s', $START_AT); push @this, sprintf('-e %s', $END_AT); push @this, sprintf('-t %s', "$name"); @@ -108,6 +125,7 @@ for my $source (@SOURCES) { push @this, sprintf('CDEF:%s=d_runningVerified,0,*,%s,+' , 'zeroline' , '0'); push @this, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); + print "Graphing $hash running.\n" if $VERBOSE; my ($averages,$xsize,$ysize) = RRDs::graph @this; warn RRDs::error if RRDs::error; @@ -118,7 +136,7 @@ for my $source (@SOURCES) { "platform $platform
". "published $published". "". - "

\"$name\n"; + "

\"$name\n"; $index{$section} .= "

  • $name
    • \n"; if (-e $source_traffic) { @@ -159,7 +177,7 @@ for my $source (@SOURCES) { for my $thumb (0, 1) { @this = (); - push @this, sprintf($RESULT_DIR.'/node-'.($thumb ? 'small-' : '').'traf-%s.png', $hash); + push @this, sprintf($RESULT_DIR.'/'.$subdir.'/node-'.($thumb ? 'small-' : '').'traf-%s.png', $hash); #push @this, sprintf('-s %s', '-604800'); push @this, sprintf('-s %s', $START_AT); push @this, sprintf('-e %s', $END_AT); @@ -204,12 +222,13 @@ for my $source (@SOURCES) { #push @this, sprintf('CDEF:%s=read,0,*,%s,+' , 'zeroline' , '0'); #push @this, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); + print "Graphing $hash traffic $thumb.\n" if $VERBOSE; ($averages,$xsize,$ysize) = RRDs::graph @this; warn RRDs::error if RRDs::error; }; - $body{$section} .= " "; - $body{$section} .= "\"$name"; + $body{$section} .= " "; + $body{$section} .= "\"$name"; $body{$section} .= ""; }; #$body{$section} .= "
    top"; @@ -253,6 +272,7 @@ sub makeTotalTrafficPic($$$) { push @totalTraffic, sprintf('CDEF:zeroline=aggread%d,0,*', $traf_item_counter); push @totalTraffic, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); + print "Graphing total traffic $file $age $include15min.\n" if $VERBOSE; my ($averages,$xsize,$ysize) = RRDs::graph @totalTraffic; warn RRDs::error if RRDs::error; @@ -307,6 +327,7 @@ sub makeNumbersGraph($$) { + print "Graphing numbers $file $age.\n" if $VERBOSE; my ($averages,$xsize,$ysize) = RRDs::graph @params; warn RRDs::error if RRDs::error; diff --git a/other/tor/bin/rrd-update b/other/tor/bin/rrd-update index c7062f7..30e5466 100755 --- a/other/tor/bin/rrd-update +++ b/other/tor/bin/rrd-update @@ -7,7 +7,7 @@ use Digest::SHA1 qw(sha1_hex); use Time::ParseDate; my $NOW = time; -my $VERBOSE = 1; +my $VERBOSE = 0; my $RRD = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/running_Routers.rrd'; my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes'; my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index'; @@ -85,15 +85,32 @@ sub get_last($) { check_exists_running($RRD, 1); my $last = get_last($RRD); -opendir(DIR, $RRD_DIR) || die ("Cannot opendir $RRD_DIR: $!\n"); -my @rrdfiles = grep { /\.rrd$/ } readdir (DIR); -closedir(DIR); my %last_node; -for my $rrdfile (@rrdfiles) { - my $nodename = $rrdfile; - $nodename =~ s/\.rrd$//; - $last_node{$nodename} = get_last($RRD_DIR.'/'.$rrdfile); -} +sub fetch_last_node($) { + my ($hash) = @_; + + my ($subdir) = $hash =~ /^([A-F0-9]{2})/; + die "Unknown subdir for router $hash" unless defined $subdir; + return unless -d $RRD_DIR.'/'.$subdir; + my $rrd = $RRD_DIR.'/'.$subdir.'/'.$hash.'.rrd'; + $last_node{$hash} = get_last($rrd); + $rrd = $RRD_DIR.'/'.$subdir.'/TRAF-'.$hash.'.rrd'; + $last_node{'TRAF-'.$hash} = get_last($rrd); +}; +#opendir(DIR, $RRD_DIR) || die ("Cannot opendir $RRD_DIR: $!\n"); +#my @subdirs = grep { -d $RRD_DIR.'/'.$_ && $_ =~ /^[A-F0-9]{2}$/} readdir (DIR); +#closedir(DIR); + +#for my $dir (@subdirs) { +# opendir(DIR, $RRD_DIR.'/'.$dir) || die ("Cannot opendir $RRD_DIR/$dir: $!\n"); +# my @rrdfiles = grep { /\.rrd$/ } readdir (DIR); +# closedir(DIR); +# for my $rrdfile (@rrdfiles) { +# my $nodename = $rrdfile; +# $nodename =~ s/\.rrd$//; +# $last_node{$nodename} = get_last($RRD_DIR.'/'.$dir.'/'.$rrdfile); +# }; +#}; my @dirfiles; if (scalar @ARGV) { @@ -169,9 +186,12 @@ for my $dir (@dirfiles) { my $current_write_history = undef; my $current_read_history = undef; my $current_port80 = undef; + my $last_was_empty = 0; while () { $linenr++; chomp; + next if ($last_was_empty == 1 && /^\s*$/); + $last_was_empty = 0; if (/^router\s+(\S+)\s/) { if (!defined $current_router) { $current_router = $1; @@ -315,6 +335,8 @@ for my $dir (@dirfiles) { $current_hash = undef; $current_write_history = undef; $current_read_history = undef; + + $last_was_empty = 1 } } close (DIRECTORY); @@ -367,6 +389,7 @@ for my $dir (@dirfiles) { push @fast_verified_routers_port80, $hash if ($router_desc{$hash}->{'port80'} && $fast); } + fetch_last_node($hash) unless (defined $last_node{$hash}); if (!defined $last_node{$hash} || $dir_published > $last_node{$hash}) { push @{$updates{$hash}}, $dir_published.':'. ($router =~ /^\$/ ? '0:1' : '1:0').':'. @@ -394,7 +417,7 @@ for my $dir (@dirfiles) { if (defined $hist) { # 2004-09-26 01:45:45 (900 s) 9619165,12880477,10591411... my ($time, $interval, $values) = - $hist =~ /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \((\d+) s\)\s?((?:\d+,)*\d+)?$/; + $hist =~ /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \((\d+) s\)\s?((?:-?\d+,)*-?\d+)?$/; warn ("Could not parse history '$hist'\n"),next unless defined $time; my $stamp = parsedate($time); @@ -403,11 +426,15 @@ for my $dir (@dirfiles) { next unless defined $values; my @values = split /,/, $values; for my $v (@values) { + if ($v < 0) { + $v += 4294967296; # work around bug in tor, that has history as signed 32 bit integer. + }; $hist{$stamp}->{$inout} = $v; $stamp -= $interval; }; }; }; + fetch_last_node($hash) unless (defined $last_node{'TRAF-'.$hash}); for my $stamp (sort {$a <=> $b} keys %hist) { if (!defined $last_node{'TRAF-'.$hash} || $stamp > $last_node{'TRAF-'.$hash}) { push @{$updates_traffic{$hash}}, $stamp.':'. @@ -428,7 +455,9 @@ if (scalar @updateGlobal != 0) { for my $router (keys %updates) { next unless exists $hashes_router{$router}; - my $rrd = $RRD_DIR.'/'.$router.'.rrd'; + my ($subdir) = substr($router,0,2); + mkdir($RRD_DIR.'/'.$subdir) unless -d $RRD_DIR.'/'.$subdir; + my $rrd = $RRD_DIR.'/'.$subdir.'/'.$router.'.rrd'; check_exists_running($rrd, 0); RRDs::update( $rrd, @{$updates{$router}} ); my $err=RRDs::error; @@ -441,7 +470,9 @@ for my $router (keys %updates) { for my $router (keys %updates_traffic) { next unless exists $hashes_router{$router}; - my $rrd = $RRD_DIR.'/TRAF-'.$router.'.rrd'; + my ($subdir) = substr($router,0,2); + mkdir($RRD_DIR.'/'.$subdir) unless -d $RRD_DIR.'/'.$subdir; + my $rrd = $RRD_DIR.'/'.$subdir.'/TRAF-'.$router.'.rrd'; check_exists_traffic($rrd); RRDs::update( $rrd, @{$updates_traffic{$router}} ); my $err=RRDs::error; @@ -455,7 +486,9 @@ for my $router (keys %updates_traffic) { for my $hash (keys %hashes_router) { - my $f = $INDEX_DIR.'/'.$hash.".name"; + my ($subdir) = substr($hash,0,2); + mkdir($INDEX_DIR.'/'.$subdir) unless -d $INDEX_DIR.'/'.$subdir; + my $f = $INDEX_DIR.'/'.$subdir.'/'.$hash.".name"; open(I, ">$f") or warn ("Cannot open $f: $!\n"), next; print I $hashes_router{$hash}->{'name'},"\n"; print I $hashes_router{$hash}->{'platform'},"\n"; -- cgit v1.2.3