From 4b2c75bbbf59f9b2cf9c521dd53932c156a32fa8 Mon Sep 17 00:00:00 2001 From: Peter Palfrader Date: Mon, 11 Apr 2005 18:57:23 +0000 Subject: Add load-update --- bin/load-update | 187 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100755 bin/load-update (limited to 'bin') diff --git a/bin/load-update b/bin/load-update new file mode 100755 index 0000000..6abb698 --- /dev/null +++ b/bin/load-update @@ -0,0 +1,187 @@ +#!/usr/bin/perl -w + +use strict; +use RRDs; +use BER; +use English; +use Time::ParseDate; + +my $ECHOLOT_BASE = shift @ARGV; +my $RRDPATH = shift @ARGV; +die ("Usage: $PROGRAM_NAME \n") if (!defined $ECHOLOT_BASE || !defined $RRDPATH || scalar @ARGV > 0); + +my $HOSTNAME = `hostname`; + + +my @MONTH = qw{Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec}; +my $BASEDIR = $ECHOLOT_BASE.'/results/thesaurus/'; # we depend on that trailing / +my $INDEX = $BASEDIR.'index.txt'; +my $NOW = time; +my $VERBOSE = 0; + +chdir $RRDPATH or die ("Cannot chdir $RRDPATH: $!\n"); + +sub check_exists($) { + my ($name) = @_; + my $rrd = "$name.rrd"; + return if (-e $rrd); + my @params = ($rrd); + push @params, '-b', 'now - 5 months', + qw{ --step 86400 + DS:total:GAUGE:172800:U:U + DS:typeI:GAUGE:172800:U:U + DS:typeII:GAUGE:172800:U:U + RRA:AVERAGE:0.5:1:3650 + }; + RRDs::create @params; + my $err=RRDs::error; + warn "ERROR while creating $rrd: $err\n" if $err; +} + +sub check_exists_PoolSize($) { + my ($name) = @_; + my $rrd = "$name.pool.rrd"; + return if (-e $rrd); + my @params = ($rrd); + push @params, '-b', 'now - 5 months', + qw{ --step 86400 + DS:poolSize:GAUGE:172800:0:U + RRA:AVERAGE:0.5:1:3650 + RRA:MIN:0.5:1:3650 + RRA:MAX:0.5:1:3650 + }; + RRDs::create @params; + my $err=RRDs::error; + warn "ERROR while creating $rrd: $err\n" if $err; +} + +my %REMAILERS; +open(INDEX, $INDEX) or die ("Cannot open $INDEX: $!\n"); +while () { + my($nick, $id) = $_ =~ /^([a-z0-9]+) \s+ ([0-9]+) \s/x; + next unless defined $nick; + $REMAILERS{$id} = $nick; +}; +close(INDEX); + +my %RELIABLE; +my %MIXMASTER; +for my $id (keys %REMAILERS) { + my $nick = $REMAILERS{$id}; + unless (open(F, $BASEDIR.$id.'.stats')) { + warn("Cannot open $id.stats: $!\n") if ($VERBOSE >= 1); + next; + }; + local $/; + undef $/; + my $stats = ; + close F; + + if ($stats =~ /^Number of messages per day:/m) { + $MIXMASTER{$id} = $stats; + } elsif ($stats =~ /^\s*Messages Received Last Week:\s+([0-9]+)/m) { + my $total = $1; + my ($generated) = $stats =~ /^\s*Generated: (.*)/m; + if (defined $generated) { + my $gen = parsedate($generated); + unless (defined $generated) { + print "Cannot parse generated '$generated' for $id\n"; + }; + $generated = $gen; + #print scalar gmtime $generated,"\n"; + }; + unless (defined $generated) { # no else as if block may set generated to undef. + print "Don't have a generated date for $id.\n"; + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, + $atime,$mtime,$ctime,$blksize,$blocks) + = stat($BASEDIR.$id.'.stats'); + unless (defined $mtime) { + print "Don't even have an mtime for the file\n"; + next; + }; + $generated = $mtime; + } + $RELIABLE{$id} = { total => $total, generated => $generated}; + } else { + print "Don't know what $id is.\n"; + } +} + + +# do reliable +# find out last week: +my $last_week = $NOW - ($NOW % (3600*24*7)) - 3600*24*7; +for my $id (keys %RELIABLE) { + my $remailer = $RELIABLE{$id}; + check_exists($REMAILERS{$id}); + my $last = RRDs::last($REMAILERS{$id}.'.rrd'); + my $err=RRDs::error; + warn "ERROR while updating: $err\n" if $err; + my $time = $remailer->{'generated'} - ($remailer->{'generated'} % (3600*24)); + next if ($time <= $last); + RRDs::update( $REMAILERS{$id}.'.rrd' , $time.':'.int($remailer->{'total'}/7).':U:U' ); + $err=RRDs::error; + warn "ERROR while updating: $err\n" if $err; +} + + # Number of messages per day: + # 16 Aug: 14691 Intermediate:9568 Mail:4637 Postings: 24 + # 17 Aug: 717 Intermediate: 579 Mail: 104 Postings: 0 + +for my $id (keys %MIXMASTER) { + my @updates; + my @updatesPoolSize; + check_exists($REMAILERS{$id}); + + my $last = 0; + if ( -e $REMAILERS{$id}.'.rrd') { + $last = RRDs::last($REMAILERS{$id}.'.rrd'); + my $err=RRDs::error; + warn "ERROR while updating: $err\n" if $err; + }; + my $lastPoolSize = 0; + if (-e $REMAILERS{$id}.'.pool.rrd') { + $lastPoolSize = RRDs::last($REMAILERS{$id}.'.pool.rrd'); + my $err=RRDs::error; + warn "ERROR while updating: $err\n" if $err; + }; + + # start 4 months ago: + for (my $time = $NOW - ($NOW % (3600*24)) - 3600*24*4*30; $time < $NOW; $time += 3600*24) { + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday) = gmtime($time); + my $match = sprintf("%02d %s:", $mday, $MONTH[$mon]); + + my ($type1, $type2, $total, $pool); + if ($MIXMASTER{$id} =~ /^$match\s*([0-9]+)(?:.*Pool\s*size:\s*([0-9]+))?/m) { + $type2 = $total = $1; + $type1 = 'U'; + $pool = defined $2 ? $2 : undef; + } elsif ($MIXMASTER{$id} =~ /^$match\s+Mix:\s*([0-9]+)\s* PGP:\s*([0-9]+)(?:.*Pool\s*size:\s*([0-9]+))?/m) { + $type1 = $2; + $type2 = $1; + $total = $1 + $2; + $pool = defined $3 ? $3 : undef; + } else { + next; + }; + + if ($time > $last) { + push @updates, $time.':'.$total.':'.$type1.':'.$type2; + }; + if (defined $pool && $time > $lastPoolSize) { + push @updatesPoolSize, $time.':'.$pool; + }; + } + if (scalar @updates != 0) { + check_exists($REMAILERS{$id}); + RRDs::update( $REMAILERS{$id}.'.rrd', @updates ); + my $err=RRDs::error; + warn "ERROR while updating: $err\n" if $err; + }; + if (scalar @updatesPoolSize != 0) { + check_exists_PoolSize($REMAILERS{$id}); + RRDs::update( $REMAILERS{$id}.'.pool.rrd', @updatesPoolSize ); + my $err=RRDs::error; + warn "ERROR while updating: $err\n" if $err; + }; +} -- cgit v1.2.3