Add load-update

1 files changed, 187 insertions, 0 deletions

diff --git a/bin/load-update b/bin/load-update
new file mode 100755
index 0000000..6abb698
--- /dev/null
+++ b/bin/load-update
@@ -0,0 +1,187 @@
+#!/usr/bin/perl -w
+
+use strict;
+use RRDs;
+use BER;
+use English;
+use Time::ParseDate;
+
+my $ECHOLOT_BASE = shift @ARGV;
+my $RRDPATH = shift @ARGV;
+die ("Usage: $PROGRAM_NAME <echolot base> <rrd dir>\n") if (!defined $ECHOLOT_BASE || !defined $RRDPATH || scalar @ARGV > 0);
+
+my $HOSTNAME = `hostname`;
+
+
+my @MONTH = qw{Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec};
+my $BASEDIR = $ECHOLOT_BASE.'/results/thesaurus/';  # we depend on that trailing /
+my $INDEX = $BASEDIR.'index.txt';
+my $NOW = time;
+my $VERBOSE = 0;
+
+chdir $RRDPATH or die ("Cannot chdir $RRDPATH: $!\n");
+
+sub check_exists($) {
+	my ($name) = @_;
+	my $rrd = "$name.rrd";
+	return if (-e $rrd);
+	my @params = ($rrd);
+	push @params, '-b', 'now - 5 months',
+	  qw{ --step 86400
+	      DS:total:GAUGE:172800:U:U
+	      DS:typeI:GAUGE:172800:U:U
+	      DS:typeII:GAUGE:172800:U:U
+	      RRA:AVERAGE:0.5:1:3650
+	};
+	RRDs::create @params;
+	my $err=RRDs::error;
+	warn "ERROR while creating $rrd: $err\n" if $err;
+}
+
+sub check_exists_PoolSize($) {
+	my ($name) = @_;
+	my $rrd = "$name.pool.rrd";
+	return if (-e $rrd);
+	my @params = ($rrd);
+	push @params, '-b', 'now - 5 months',
+	  qw{ --step 86400
+	      DS:poolSize:GAUGE:172800:0:U
+	      RRA:AVERAGE:0.5:1:3650
+	      RRA:MIN:0.5:1:3650
+	      RRA:MAX:0.5:1:3650
+	};
+	RRDs::create @params;
+	my $err=RRDs::error;
+	warn "ERROR while creating $rrd: $err\n" if $err;
+}
+
+my %REMAILERS;
+open(INDEX, $INDEX) or die ("Cannot open $INDEX: $!\n");
+while (<INDEX>) {
+	my($nick, $id) = $_ =~ /^([a-z0-9]+)  \s+ ([0-9]+) \s/x;
+	next unless defined $nick;
+	$REMAILERS{$id} = $nick;
+};
+close(INDEX);
+
+my %RELIABLE;
+my %MIXMASTER;
+for my $id (keys %REMAILERS) {
+	my $nick = $REMAILERS{$id};
+	unless (open(F, $BASEDIR.$id.'.stats')) {
+		warn("Cannot open $id.stats: $!\n") if ($VERBOSE >= 1);
+		next;
+	};
+	local $/;
+	undef $/;
+	my $stats = <F>;
+	close F;
+
+	if ($stats =~ /^Number of messages per day:/m) {
+		$MIXMASTER{$id} = $stats;
+	} elsif ($stats =~ /^\s*Messages Received Last Week:\s+([0-9]+)/m) {
+		my $total = $1;
+		my ($generated) = $stats =~ /^\s*Generated: (.*)/m;
+		if (defined $generated) {
+			my $gen = parsedate($generated);
+			unless (defined $generated) {
+				print "Cannot parse generated '$generated' for $id\n";
+			};
+			$generated = $gen;
+			#print scalar gmtime $generated,"\n";
+		};
+		unless (defined $generated) { # no else as if block may set generated to undef.
+			print "Don't have a generated date for $id.\n";
+			my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
+			    $atime,$mtime,$ctime,$blksize,$blocks)
+			   = stat($BASEDIR.$id.'.stats');
+			unless (defined $mtime) {
+				print "Don't even have an mtime for the file\n";
+				next;
+			};
+			$generated = $mtime;
+		}
+		$RELIABLE{$id} = { total => $total, generated => $generated};
+	} else {
+		print "Don't know what $id is.\n";
+	}
+}
+
+
+# do reliable
+# find out last week:
+my $last_week = $NOW - ($NOW % (3600*24*7)) - 3600*24*7;
+for my $id (keys %RELIABLE) {
+	my $remailer = $RELIABLE{$id};
+	check_exists($REMAILERS{$id});
+	my $last = RRDs::last($REMAILERS{$id}.'.rrd');
+	my $err=RRDs::error;
+	warn "ERROR while updating:  $err\n" if $err;
+	my $time = $remailer->{'generated'} - ($remailer->{'generated'} % (3600*24));
+	next if ($time <= $last);
+	RRDs::update( $REMAILERS{$id}.'.rrd' , $time.':'.int($remailer->{'total'}/7).':U:U' );
+	$err=RRDs::error;
+	warn "ERROR while updating:  $err\n" if $err;
+}
+
+	# Number of messages per day:
+	# 16 Aug: 14691  Intermediate:9568  Mail:4637  Postings:  24
+	# 17 Aug:  717  Intermediate: 579  Mail: 104  Postings:   0
+
+for my $id (keys %MIXMASTER) {
+	my @updates;
+	my @updatesPoolSize;
+	check_exists($REMAILERS{$id});
+
+	my $last = 0;
+	if ( -e $REMAILERS{$id}.'.rrd') {
+		$last = RRDs::last($REMAILERS{$id}.'.rrd');
+		my $err=RRDs::error;
+		warn "ERROR while updating:  $err\n" if $err;
+	};
+	my $lastPoolSize = 0;
+	if (-e $REMAILERS{$id}.'.pool.rrd') {
+		$lastPoolSize = RRDs::last($REMAILERS{$id}.'.pool.rrd');
+		my $err=RRDs::error;
+		warn "ERROR while updating:  $err\n" if $err;
+	};
+
+	# start 4 months ago:
+	for (my $time = $NOW - ($NOW % (3600*24)) - 3600*24*4*30; $time < $NOW; $time += 3600*24) {
+		my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday) = gmtime($time);
+		my $match = sprintf("%02d %s:", $mday, $MONTH[$mon]);
+
+		my ($type1, $type2, $total, $pool);
+		if ($MIXMASTER{$id} =~ /^$match\s*([0-9]+)(?:.*Pool\s*size:\s*([0-9]+))?/m) {
+			$type2 = $total = $1;
+			$type1 = 'U';
+			$pool = defined $2 ? $2 : undef;
+		} elsif ($MIXMASTER{$id} =~ /^$match\s+Mix:\s*([0-9]+)\s* PGP:\s*([0-9]+)(?:.*Pool\s*size:\s*([0-9]+))?/m) {
+			$type1 = $2;
+			$type2 = $1;
+			$total = $1 + $2;
+			$pool = defined $3 ? $3 : undef;
+		} else {
+			next;
+		};
+
+		if ($time > $last) {
+			push @updates, $time.':'.$total.':'.$type1.':'.$type2;
+		};
+		if (defined $pool && $time > $lastPoolSize) {
+			push @updatesPoolSize, $time.':'.$pool;
+		};
+	}
+	if (scalar @updates != 0) {
+		check_exists($REMAILERS{$id});
+		RRDs::update( $REMAILERS{$id}.'.rrd', @updates );
+		my $err=RRDs::error;
+		warn "ERROR while updating:  $err\n" if $err;
+	};
+	if (scalar @updatesPoolSize != 0) {
+		check_exists_PoolSize($REMAILERS{$id});
+		RRDs::update( $REMAILERS{$id}.'.pool.rrd', @updatesPoolSize );
+		my $err=RRDs::error;
+		warn "ERROR while updating:  $err\n" if $err;
+	};
+}