Repository List / perl / trunk / rrd / log2rrd.pl @ HEAD

#!/usr/local/bin/perl -w

# Convert http access log to rrdtool database
# 2002-08-01 Dobrica Pavlinisic, <dpavlin@rot13.org>
#
# based on work of Jean-Edouard BABIN (listes@jeb.com.fr)
# http://groups.google.com/groups?hl=en&lr=&ie=UTF-8&safe=off&selm=aap7n0%241slf%241%40FreeBSD.csie.NCTU.edu.tw
# 
# Usage: log2rrd.pl < access.log [2> errors]
#
# there is one counter called tmp which can be used to graph some
# perticual URL hit (see below in code how to edit it).

use strict;
use RRDs;
use Date::Parse;

# feed data how often to rrdtool (seconds)
my $step = 300;
# name of rrdfile to create
my $rrd_file = "access";

my $counter = 0;
my ($dv, $sv, $ov, $tmp) = (0, 0, 0);

my $last_utime = 0;
my $utime = 0;

print "   all\t   dyn\t   sta\t   oth   tmp\n";

while (<STDIN>) {
	if ($utime - $last_utime > $step) {
		printf "%7d %7d %7d %7d %7d\n", $counter, $dv, $sv, $ov, $tmp;
		&create_rrd("$rrd_file.rrd") unless -f "$rrd_file.rrd";
		RRDs::update("$rrd_file.rrd", "--template",
			"dynamic_views:static_views:other_views:tmp",
			"$utime:$dv:$sv:$ov:$tmp");
		my $ERR=RRDs::error;
		die "ERROR while updating $rrd_file.rrd: $ERR\n" if $ERR;
		($dv, $sv, $ov, $tmp) = (0, 0, 0, 0);
		$last_utime = $utime;
	} else {
		my $request = $_;
		if ($request =~ m/\[([^\]]+)\]/) {
			$utime = str2time($1);
		} else {
			print STDERR "can't parse date in line '$request'\n";
			next;
		}
		my $url;
		if ($request =~ m,\"((GET)|(POST))\s+(\S+)\s+HTTP,) {
			$url = $4;
		} else {
			print STDERR "can't parse request in line '$request'\n";
			next;
		}

		my ($page, undef) = split(m/\?/, $url);
		if ($page =~ m/(cgi-bin\/|\.php$|\.pl$|\.sh$)/ ) {
			$dv++;		# dynamic content
		} elsif ($page =~ m/\.html?$/) {
			$sv++;		# static content
		} else {
			$ov++;		# image/other file types
		}

		# tmp counter specification
		if ($page =~ m,/cgi-bin/search/search.pl,) {
			$tmp++;
		}

		$counter++;
	}
}

sub create_rrd {
	my($file) = @_;

	print "creating rrd database $file (step: $step utime: $utime)\n";

	RRDs::create($file,
		"--step", $step,	# we're feeding data every $step seconds
		"--start", $utime - $step,	# start at current utime

		"DS:dynamic_views:ABSOLUTE:600:0:U",
		"DS:static_views:ABSOLUTE:600:0:U",
		"DS:other_views:ABSOLUTE:600:0:U",
		"DS:tmp:ABSOLUTE:600:0:U",

		"RRA:AVERAGE:0.5:1:600",

		"RRA:AVERAGE:0.5:6:600",
		"RRA:AVERAGE:0.5:24:600",
		"RRA:AVERAGE:0.5:288:600",
		
		"RRA:MAX:0.5:6:600",
		"RRA:MAX:0.5:24:600",
		"RRA:MAX:0.5:288:600",

		"RRA:LAST:0.5:1:1");		# keep latest data point

	my $ERR=RRDs::error;
	die "ERROR while updating $rrd_file.rrd: $ERR\n" if $ERR;
}