#!/usr/bin/perl -w use strict; use RRDs; my $HOSTNAME = `hostname`; #my $IMG_URL = 'http://images.noreply.org/tor-running-routers/'; #my $IMG_URL = 'http://new.noreply.org/tor-running-routers/'; my $IMG_URL = ''; my $NOW = time; my $GMTIME = gmtime($NOW); my $START_AT = $NOW-4*30*24*3600; my $END_AT = $NOW-4*3600; my $OLD_AT = $NOW-7*24*3600; my $RRD = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/running_Routers.rrd'; my $RRD_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/rrd/nodes'; my $INDEX_DIR = '/home/weasel/www/www.noreply.org/Build/other/tor/index'; my $RESULT_DIR = 'result.new'; my $LONG_TIME_MONTHS = 24; my $VERBOSE = 0; # Get list of RRD files opendir(D,$RRD_DIR) or die ("Cannot opendir $RRD_DIR: $!\n"); my @dirs = grep { -d $RRD_DIR.'/'.$_ && $_ =~ /^[A-F0-9]{2}$/ } readdir(D); closedir(D); my @SOURCES; for my $dir (@dirs) { opendir(D,$RRD_DIR.'/'.$dir) or die ("Cannot opendir $RRD_DIR/$dir: $!\n"); my @sources = grep { /\.rrd$/ } readdir(D); @sources = grep { !/^TRAF-/ } @sources; @sources = grep { my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, $atime,$mtime,$ctime,$blksize,$blocks) = stat($RRD_DIR.'/'.$dir.'/'.$_); $mtime >= $START_AT; } @sources; push @SOURCES, @sources; closedir(D); }; # Read hash <-> name mapping my %hashes_router; opendir(D,$INDEX_DIR) or die ("Cannot opendir $INDEX_DIR: $!\n"); @dirs = grep { -d $INDEX_DIR.'/'.$_ && $_ =~ /^[A-F0-9]{2}$/ } readdir(D); closedir(D); for my $dir (@dirs) { opendir(D,$INDEX_DIR.'/'.$dir) or die ("Cannot opendir $INDEX_DIR/$dir: $!\n"); my @NAME = grep { /\.name$/ } readdir(D); closedir(D); for my $hash (@NAME) { my $f = $INDEX_DIR.'/'.$dir.'/'.$hash; open (F, $f) or warn("Cannot open $f: $!\n"), next; my $name = ; my $platform = ; my $published = ; chomp $name; chomp $platform; chomp $published; close F; $hash =~ s/\.name$//; $hashes_router{$hash} = {name => $name, platform => $platform, published => $published }; }; }; @SOURCES = sort { my $aa = $a; my $bb = $b; $aa =~ s/\.rrd//; $bb =~ s/\.rrd//; lc($hashes_router{$aa}->{'name'}) cmp lc($hashes_router{$bb}->{'name'}) || $a cmp $b } @SOURCES; my @nodesTraffic = (); my %body; my %index; my $traf_item_counter = 0; for my $source (@SOURCES) { my $hash = $source; $hash =~ s/\.rrd$//; my $name = $hashes_router{$hash}->{'name'}; die ("No name for $hash") unless defined $name; my $platform = $hashes_router{$hash}->{'platform'}; my $published = $hashes_router{$hash}->{'published'}; my $def = $name; my $cdef = 'c_'.$def; my $label = $def; my $subdir = substr($hash,0,2); my $source_traffic = $RRD_DIR .'/'.$subdir.'/TRAF-'. $source; $source = $RRD_DIR .'/'.$subdir.'/'. $source; my (undef,undef,undef,undef,undef,undef,undef,undef, undef,$mtime,undef,undef,undef) = stat($source); my $section = $mtime < $OLD_AT ? 'old' : 'current'; my @this; mkdir ($RESULT_DIR.'/'.$subdir) unless -d ($RESULT_DIR.'/'.$subdir); push @this, sprintf($RESULT_DIR.'/'.$subdir.'/node-%s.png', $hash); push @this, sprintf('-s %s', $START_AT); push @this, sprintf('-e %s', $END_AT); push @this, sprintf('-t %s', "$name"); push @this, sprintf('-aPNG'); push @this, sprintf('-v %s', "running"); push @this, sprintf('-w %d', 150); push @this, sprintf('-h %d', 50); push @this, sprintf('DEF:%s=%s:%s:%s', 'd_runningVerified' , $source, 'runningVerified', 'AVERAGE'); push @this, sprintf('CDEF:%s=%s' , 'c_runningVerified' , 'd_runningVerified'); push @this, sprintf('DEF:%s=%s:%s:%s', 'd_runningUnverified' , $source, 'runningUnverified', 'AVERAGE'); push @this, sprintf('CDEF:%s=%s' , 'c_runningUnverified' , 'd_runningUnverified'); push @this, sprintf('DEF:%s=%s:%s:%s', 'd_exit80Verified' , $source, 'exit80Verified', 'AVERAGE'); push @this, sprintf('CDEF:c_exit80Verified=d_exit80Verified,0.5,*,c_runningVerified,*'); push @this, sprintf('%s:%s#%s' , 'AREA' , 'c_runningVerified' , '7777FF' ); push @this, sprintf('%s:%s#%s' , 'STACK' , 'c_runningUnverified' , 'FF7700' ); push @this, sprintf('%s:%s#%s' , 'AREA' , 'c_exit80Verified' , '00FF00' ); push @this, sprintf('CDEF:%s=d_runningVerified,0,*,%s,+' , 'zeroline' , '0'); push @this, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); print "Graphing $hash running.\n" if $VERBOSE; my ($averages,$xsize,$ysize) = RRDs::graph @this; warn RRDs::error if RRDs::error; $body{$section} .= "

$name

". "

". "Last descriptor data:". "

". "platform $platform
". "published $published". "
". "

\"$name\n"; $index{$section} .= "

  • $name
  • \n"; if (-e $source_traffic) { $traf_item_counter++; push @nodesTraffic, sprintf('DEF:read%d=%s:read:AVERAGE', $traf_item_counter, $source_traffic); push @nodesTraffic, sprintf('DEF:write%d=%s:write:AVERAGE', $traf_item_counter, $source_traffic); push @nodesTraffic, sprintf('DEF:max_read%d=%s:read:MAX', $traf_item_counter, $source_traffic); push @nodesTraffic, sprintf('DEF:max_write%d=%s:write:MAX', $traf_item_counter, $source_traffic); #push @nodesTraffic, sprintf('DEF:min_read%d=%s:read:MIN', $traf_item_counter, $source_traffic); #push @nodesTraffic, sprintf('DEF:min_write%d=%s:write:MIN', $traf_item_counter, $source_traffic); push @nodesTraffic, sprintf('DEF:max_capacity%d=%s:capacity:MAX', $traf_item_counter, $source); push @nodesTraffic, sprintf('CDEF:cleanread%d=read%d,UN,0,read%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); push @nodesTraffic, sprintf('CDEF:cleanwrite%d=write%d,UN,0,write%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); push @nodesTraffic, sprintf('CDEF:max_cleanread%d=max_read%d,UN,0,max_read%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); push @nodesTraffic, sprintf('CDEF:max_cleanwrite%d=max_write%d,UN,0,max_write%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); #push @nodesTraffic, sprintf('CDEF:min_cleanread%d=min_read%d,UN,0,min_read%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); #push @nodesTraffic, sprintf('CDEF:min_cleanwrite%d=min_write%d,UN,0,min_write%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); push @nodesTraffic, sprintf('CDEF:max_cleancapacity%d=max_capacity%d,UN,0,max_capacity%d,IF', $traf_item_counter, $traf_item_counter, $traf_item_counter); if ($traf_item_counter == 1) { push @nodesTraffic, sprintf('CDEF:aggread%d=cleanread%d', $traf_item_counter, $traf_item_counter); push @nodesTraffic, sprintf('CDEF:aggwrite%d=cleanwrite%d', $traf_item_counter, $traf_item_counter); push @nodesTraffic, sprintf('CDEF:max_aggread%d=max_cleanread%d', $traf_item_counter, $traf_item_counter); push @nodesTraffic, sprintf('CDEF:max_aggwrite%d=max_cleanwrite%d', $traf_item_counter, $traf_item_counter); #push @nodesTraffic, sprintf('CDEF:min_aggread%d=min_cleanread%d', $traf_item_counter, $traf_item_counter); #push @nodesTraffic, sprintf('CDEF:min_aggwrite%d=min_cleanwrite%d', $traf_item_counter, $traf_item_counter); push @nodesTraffic, sprintf('CDEF:max_aggcapacity%d=max_cleancapacity%d', $traf_item_counter, $traf_item_counter); } else { push @nodesTraffic, sprintf('CDEF:aggread%d=cleanread%d,aggread%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); push @nodesTraffic, sprintf('CDEF:aggwrite%d=cleanwrite%d,aggwrite%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); push @nodesTraffic, sprintf('CDEF:max_aggread%d=max_cleanread%d,max_aggread%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); push @nodesTraffic, sprintf('CDEF:max_aggwrite%d=max_cleanwrite%d,max_aggwrite%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); #push @nodesTraffic, sprintf('CDEF:min_aggread%d=min_cleanread%d,min_aggread%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); #push @nodesTraffic, sprintf('CDEF:min_aggwrite%d=min_cleanwrite%d,min_aggwrite%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); push @nodesTraffic, sprintf('CDEF:max_aggcapacity%d=max_cleancapacity%d,max_aggcapacity%d,+', $traf_item_counter, $traf_item_counter, $traf_item_counter-1); }; for my $thumb (0, 1) { @this = (); push @this, sprintf($RESULT_DIR.'/'.$subdir.'/node-'.($thumb ? 'small-' : '').'traf-%s.png', $hash); #push @this, sprintf('-s %s', '-604800'); push @this, sprintf('-s %s', $START_AT); push @this, sprintf('-e %s', $END_AT); push @this, sprintf('-t %s', "$name traffic"); push @this, sprintf('-v %s', "bytes/sec"); if ($thumb) { push @this, sprintf('-w %d', 150); push @this, sprintf('-h %d', 50); }; push @this, sprintf('-aPNG'); push @this, sprintf('DEF:read=%s:read:AVERAGE', $source_traffic); push @this, sprintf('DEF:maxread=%s:read:MAX', $source_traffic); #push @this, sprintf('DEF:minread=%s:read:MIN', $source_traffic); push @this, sprintf('DEF:write=%s:write:AVERAGE', $source_traffic); push @this, sprintf('DEF:maxwrite=%s:write:MAX', $source_traffic); #push @this, sprintf('DEF:minwrite=%s:write:MIN', $source_traffic); push @this, sprintf('DEF:maxcapacity=%s:capacity:MAX', $source); push @this, sprintf('AREA:read#00FF00:Read'); push @this, sprintf('LINE2:write#0000FF:Write'); unless ($thumb) { push @this, sprintf('LINE1:maxread#009A2C:Read [max 15min]'); push @this, sprintf('LINE1:maxwrite#5555FF:Write [max 15min]'); #push @this, sprintf('LINE1:minread#FF76D4:Read [min]'); #push @this, sprintf('LINE1:minwrite#222255:Write [min]'); push @this, sprintf('LINE1:maxcapacity#FF5555:Capacity'); } #push @this, sprintf('COMMENT:\\n'); #push @this, sprintf('COMMENT:%s\\r', $GMTIME); #push @this, sprintf('COMMENT:\\n'); #push @this, sprintf('COMMENT: Min Max Average Last\\n'); #push @this, sprintf('COMMENT:Read: '); #push @this, sprintf('GPRINT:minread:MIN:%6.2lf %sb/s '); #push @this, sprintf('GPRINT:maxread:MAX:%6.2lf %sb/s '); #push @this, sprintf('GPRINT:read:AVERAGE:%6.2lf %sb/s '); #push @this, sprintf('GPRINT:read:LAST:%6.2lf %sb/s\\n'); #push @this, sprintf('COMMENT:Write: '); #push @this, sprintf('GPRINT:minwrite:MIN:%6.2lf %sb/s '); #push @this, sprintf('GPRINT:maxwrite:MAX:%6.2lf %sb/s '); #push @this, sprintf('GPRINT:write:AVERAGE:%6.2lf %sb/s '); #push @this, sprintf('GPRINT:write:LAST:%6.2lf %sb/s\\n'); #push @this, sprintf('CDEF:%s=read,0,*,%s,+' , 'zeroline' , '0'); #push @this, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); print "Graphing $hash traffic $thumb.\n" if $VERBOSE; ($averages,$xsize,$ysize) = RRDs::graph @this; warn RRDs::error if RRDs::error; }; $body{$section} .= " "; $body{$section} .= "\"$name"; $body{$section} .= ""; }; #$body{$section} .= "
    top"; $body{$section} .= "
    back"; $body{$section} .= '

    '; }; sub makeTotalTrafficPic($$$) { my ($file, $age, $include15min) = @_; my @totalTraffic = (); push @totalTraffic, $file; push @totalTraffic, sprintf('-s %s', $NOW-$age); #push @totalTraffic, sprintf('-s %s', $START_AT); push @totalTraffic, sprintf('-e %s', $END_AT); #push @totalTraffic, sprintf('-e %s', $NOW- 2*24*3600); push @totalTraffic, sprintf('-t %s', "Total Traffic"); push @totalTraffic, sprintf('-aPNG'); push @totalTraffic, sprintf('-v %s', "Bandwidth Used"); #push @totalTraffic, sprintf('-w %d', 600); #push @totalTraffic, sprintf('-h %d', 400); push @totalTraffic, sprintf('-w %d', 497); push @totalTraffic, sprintf('-h %d', 201); #push @totalTraffic, sprintf('-h %d', 301); #push @totalTraffic, sprintf('-h %d', 271); push @totalTraffic, @nodesTraffic; push @totalTraffic, sprintf('AREA:aggread%d#00FF00:Read bytes/s', $traf_item_counter); push @totalTraffic, sprintf('LINE2:aggwrite%d#0000FF:Write bytes/s', $traf_item_counter); push @totalTraffic, sprintf('LINE1:max_aggread%d#009A2C:Read [max 15min]', $traf_item_counter) if $include15min; push @totalTraffic, sprintf('LINE1:max_aggwrite%d#5555FF:Write [max 15min]', $traf_item_counter) if $include15min; #push @totalTraffic, sprintf('LINE1:min_aggread%d#FF76D4:Read [min]', $traf_item_counter); #push @totalTraffic, sprintf('LINE1:min_aggwrite%d#222255:Write [min]', $traf_item_counter); push @totalTraffic, sprintf('LINE1:max_aggcapacity%d#FF5555:Capacity', $traf_item_counter); push @totalTraffic, sprintf('CDEF:zeroline=aggread%d,0,*', $traf_item_counter); push @totalTraffic, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); print "Graphing total traffic $file $age $include15min.\n" if $VERBOSE; my ($averages,$xsize,$ysize) = RRDs::graph @totalTraffic; warn RRDs::error if RRDs::error; return ($averages,$xsize,$ysize); }; sub makeNumbersGraph($$) { my ($file, $age) = @_; my @params; push @params, $file; push @params, sprintf('-s %s', $NOW-$age); push @params, sprintf('-e %s', $END_AT); #push @params, sprintf('-e %s', $NOW- 2*24*3600); push @params, sprintf('-t %s', "Running routers"); push @params, sprintf('-aPNG'); push @params, sprintf('-v %s', "# Routers"); #push @params, sprintf('-w %d', 600); #push @params, sprintf('-h %d', 400); push @params, sprintf('-w %d', 497); push @params, sprintf('-h %d', 301); #push @params, sprintf('-h %d', 271); push @params, sprintf('DEF:%s=%s:%s:%s', 'd_runningVerified' , $RRD, 'runningVerified', 'AVERAGE'); push @params, sprintf('CDEF:%s=%s' , 'c_runningVerified' , 'd_runningVerified'); push @params, sprintf('DEF:%s=%s:%s:%s', 'd_runningUnverified' , $RRD, 'runningUnverified', 'AVERAGE'); push @params, sprintf('CDEF:%s=%s' , 'c_runningUnverified' , 'd_runningUnverified'); push @params, sprintf('DEF:%s=%s:%s:%s', 'd_exit80Verified' , $RRD, 'exit80Verified', 'AVERAGE'); push @params, sprintf('CDEF:%s=%s' , 'c_exit80Verified' , 'd_exit80Verified'); push @params, sprintf('DEF:%s=%s:%s:%s', 'd_f_runningVerified' , $RRD, 'fastRunningVerified', 'AVERAGE'); push @params, sprintf('CDEF:%s=%s' , 'c_f_runningVerified' , 'd_f_runningVerified'); push @params, sprintf('DEF:%s=%s:%s:%s', 'd_f_runningUnverified' , $RRD, 'fastRunningUnverifi', 'AVERAGE'); push @params, sprintf('CDEF:%s=%s' , 'c_f_runningUnverified' , 'd_f_runningUnverified'); push @params, sprintf('DEF:%s=%s:%s:%s', 'd_f_exit80Verified' , $RRD, 'fastExit80Verified', 'AVERAGE'); push @params, sprintf('CDEF:%s=%s' , 'c_f_exit80Verified' , 'd_f_exit80Verified'); #push @params, sprintf('%s:%s#%s:%s' , 'AREA' , 'c_runningVerified' , '7777FF', 'verified Nodes' ); #push @params, sprintf('%s:%s#%s:%s' , 'STACK' , 'c_runningUnverified' , 'FF7700', 'unverified Nodes' ); #push @params, sprintf('%s:%s#%s:%s' , 'LINE3' , 'c_exit80Verified' , '00FF00', 'verified Nodes exiting to port 80' ); push @params, sprintf('%s:%s#%s:%s' , 'AREA' , 'c_runningVerified' , 'AAAAFF', 'verified Nodes' ); push @params, sprintf('%s:%s#%s:%s' , 'STACK' , 'c_runningUnverified' , 'FF7700', '+unverified Nodes' ); push @params, sprintf('%s:%s#%s:%s' , 'LINE2' , 'c_exit80Verified' , 'BBFFBB', 'verified Nodes exiting to port 80' ); push @params, sprintf('%s:%s#%s:%s' , 'LINE3' , 'c_f_runningVerified' , '0000FF', 'fast verified Nodes' ); push @params, sprintf('%s:%s#%s:%s' , 'STACK' , 'c_f_runningUnverified' , 'FF0000', '+fast unverified Nodes' ); push @params, sprintf('%s:%s#%s:%s' , 'LINE3' , 'c_f_exit80Verified' , '00FF00', 'fast verified Nodes exiting to port 80' ); push @params, sprintf('CDEF:%s=d_runningVerified,0,*,%s,+' , 'zeroline' , '0'); push @params, sprintf('%s:%s' , 'LINE1' , 'zeroline' ); print "Graphing numbers $file $age.\n" if $VERBOSE; my ($averages,$xsize,$ysize) = RRDs::graph @params; warn RRDs::error if RRDs::error; return ($averages,$xsize,$ysize); }; my $html = "Number of Running Tor routers"; $html .= "

    Number of Running Tor routers

    "; $html .= "..
    \n"; my ($averages,$xsize,$ysize); makeNumbersGraph($RESULT_DIR.'/totalWeekly.png', 7*24*3600); makeNumbersGraph($RESULT_DIR.'/totalBiWeekly.png', 14*24*3600); makeNumbersGraph($RESULT_DIR.'/totalMonthly.png', 35*24*3600); makeNumbersGraph($RESULT_DIR.'/totalLong.png', $LONG_TIME_MONTHS*30*24*3600); ($averages,$xsize,$ysize) = makeNumbersGraph($RESULT_DIR.'/total.png', $NOW - $START_AT); $html .= "

    "; $html .= "

    \"Fast\" means a capacity of over 20,000 bytes/second

    \n"; $html .= "

    Number of routers over the last $LONG_TIME_MONTHS months | "; $html .= " last month |"; $html .= " last 2 weeks |"; $html .= " last week

    "; makeTotalTrafficPic($RESULT_DIR.'/totalTrafficWeekly.png', 7*24*3600, 1); makeTotalTrafficPic($RESULT_DIR.'/totalTrafficBiWeekly.png', 14*24*3600, 1); makeTotalTrafficPic($RESULT_DIR.'/totalTrafficMonthly.png', 35*24*3600, 0); makeTotalTrafficPic($RESULT_DIR.'/totalTrafficLong.png', $LONG_TIME_MONTHS*30*24*3600, 0); ($averages,$xsize,$ysize) = makeTotalTrafficPic($RESULT_DIR.'/totalTraffic.png', $NOW - $START_AT, 0); $html .= "

    "; $html .= "

    Bandwidth usage for last $LONG_TIME_MONTHS months | "; $html .= " last month |"; $html .= " last 2 weeks |"; $html .= " last week

    "; $html .= "

    The directory is downloaded from "; $html .= "http://tor.noreply.org/tor/"; $html .= " at regular intervals.
    \n"; $html .= "Other directory sources are "; $html .= "http://moria.mit.edu:9031/tor/"; $html .= " and\n"; $html .= "http://belegost.mit.edu/tor/"; $html .= "."; $html .= "

    Recent Nodes

    "; $html .= "
      ".$index{'current'}."
    "; #$html .= $body{'current'}; $html .= "

    Older Nodes

    "; $html .= "
      ".$index{'old'}."
    "; #$html .= $body{'current'}; $html .= "


    \n"; $html .= 'Images created with Tobi Oetiker\'s rrdtool'."\n"; $html .= "
    \nBuilt at $GMTIME on $HOSTNAME
    \n"; $html .= "Peter Palfrader <web\@palfrader.org>\n"; $html .= "\n"; open (F, ">$RESULT_DIR/index.html") or die ("Cannot open index.html: $!\n"); print F $html; close F; open (F, ">$RESULT_DIR/current.html") or die ("Cannot open current.html: $!\n"); print F "Current Tor Routers", "

    Current Tor routers

    ", "./
    \n", $body{'current'}, "


    \n", 'Images created with Tobi Oetiker\'s rrdtool'."\n", "
    \nBuilt at $GMTIME on $HOSTNAME
    \n", "Peter Palfrader <web\@palfrader.org>\n", "\n"; close F; open (F, ">$RESULT_DIR/old.html") or die ("Cannot open old.html: $!\n"); print F "Older Tor Routers", "

    Older Tor routers

    ", "./
    \n", $body{'old'}, "


    \n", 'Images created with Tobi Oetiker\'s rrdtool'."\n", "
    \nBuilt at $GMTIME on $HOSTNAME
    \n", "Peter Palfrader <web\@palfrader.org>\n", "\n"; close F; my %imgs = ( totalLong => 'Number of routers over the last '.$LONG_TIME_MONTHS.' months', totalMonthly => 'Number of routers over the last month', totalBiWeekly => 'Number of routers over the last 2 weeks', totalWeekly => 'Number of routers over the last week', totalTrafficLong => 'Traffic over the last '.$LONG_TIME_MONTHS.' months', totalTrafficMonthly => 'Traffic over the last month', totalTrafficBiWeekly => 'Traffic over the last 2 weeks', totalTrafficWeekly => 'Traffic over the last week' ); for my $key (keys %imgs) { open (F, ">$RESULT_DIR/$key.html") or die ("Cannot open $key.html: $!\n"); print F "$imgs{$key}", "

    $imgs{$key}

    ", "back", "

    ", "", "


    \n", "


    \nBuilt at $GMTIME on $HOSTNAME
    \n", "Peter Palfrader <web\@palfrader.org>\n", "\n"; close F; };