/[rrd-simple-monitoring]/bin/rrd-server.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /bin/rrd-server.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3 - (hide annotations)
Thu Jul 16 19:59:49 2009 UTC (14 years, 9 months ago) by dpavlin
File MIME type: text/plain
File size: 16446 byte(s)
added command-line argument -r hostname to refresh graphs

1 dpavlin 3 #!/usr/bin/perl
2 dpavlin 1 ############################################################
3     #
4     # $Id: rrd-server.pl 1092 2008-01-23 14:23:51Z nicolaw $
5     # rrd-server.pl - Data gathering script for RRD::Simple
6     #
7     # Copyright 2006, 2007, 2008 Nicola Worthington
8     #
9     # Licensed under the Apache License, Version 2.0 (the "License");
10     # you may not use this file except in compliance with the License.
11     # You may obtain a copy of the License at
12     #
13     # http://www.apache.org/licenses/LICENSE-2.0
14     #
15     # Unless required by applicable law or agreed to in writing, software
16     # distributed under the License is distributed on an "AS IS" BASIS,
17     # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18     # See the License for the specific language governing permissions and
19     # limitations under the License.
20     #
21     ############################################################
22     # vim:ts=4:sw=4:tw=78
23    
24     BEGIN {
25     # User defined constants
26 dpavlin 3 use constant BASEDIR => '/var/www/rrd';
27 dpavlin 1 use constant THEME => ('BACK#F5F5FF','SHADEA#C8C8FF','SHADEB#9696BE',
28     'ARROW#61B51B','GRID#404852','MGRID#67C6DE');
29     }
30    
31    
32    
33     BEGIN {
34     # Ensure we can find RRDs.so for RRDs.pm
35     eval "use RRDs";
36     if ($@ && !defined $ENV{LD_LIBRARY_PATH}) {
37     $ENV{LD_LIBRARY_PATH} = BASEDIR.'/lib';
38     exec($0,@ARGV);
39     }
40     }
41    
42     use 5.004;
43     use strict;
44     use warnings;
45     use lib qw(../lib);
46     use RRD::Simple 1.41;
47     use RRDs;
48     use Memoize;
49     use Getopt::Std qw();
50     use File::Basename qw(basename);
51     use File::Path qw();
52     use Config::General qw();
53     use File::Spec::Functions qw(catfile catdir);
54     use vars qw($VERSION);
55    
56     $VERSION = '1.43' || sprintf('%d', q$Revision: 1092 $ =~ /(\d+)/g);
57    
58     # Get command line options
59     my %opt = ();
60     $Getopt::Std::STANDARD_HELP_VERSION = 1;
61     $Getopt::Std::STANDARD_HELP_VERSION = 1;
62 dpavlin 3 Getopt::Std::getopts('r:u:G:T:gthvVf?', \%opt);
63 dpavlin 1
64     $opt{g} ||= $opt{G};
65     $opt{t} ||= $opt{T};
66    
67     # Display help or version
68     (VERSION_MESSAGE() && exit) if defined $opt{v};
69     (HELP_MESSAGE() && exit) if defined $opt{h} || defined $opt{'?'} ||
70 dpavlin 3 !(defined $opt{u} || defined $opt{g} || defined $opt{t} || defined $opt{r});
71 dpavlin 1
72     # cd to the righr location and define directories
73     chdir BASEDIR || die sprintf("Unable to chdir to '%s': %s", BASEDIR, $!);
74     my %dir = map { ( $_ => BASEDIR."/$_" ) } qw(bin data etc graphs cgi-bin thumbnails);
75    
76     # Create an RRD::Simple object
77     my $rrd = RRD::Simple->new(rrdtool => "$dir{bin}/rrdtool");
78    
79     # Cache results from read_create_data()
80     memoize('read_create_data');
81     memoize('read_graph_data');
82     memoize('basename');
83     memoize('graph_def');
84    
85 dpavlin 3 if ( my $hostname = $opt{r} ) {
86     warn "# refresh $hostname\n";
87     $opt{V} = 1;
88     create_thumbnails($rrd,\%dir,$hostname);
89     create_graphs($rrd,\%dir,$hostname);
90     exit;
91     }
92    
93 dpavlin 1 # Update the RRD if we've been asked to
94     my $hostname = defined $opt{u} ? update_rrd($rrd,\%dir,$opt{u}) : undef;
95    
96     # Generate some graphs
97     my @hosts;
98     for my $host (($hostname, $opt{G}, $opt{T})) {
99     next unless defined $host;
100     for (split(/\s*[,:]\s*/,$host)) {
101     push(@hosts, $_) if defined($_) && length($_);
102     }
103     }
104     @hosts = list_dir($dir{data}) unless @hosts;
105    
106     for my $hostname (@hosts) {
107     create_thumbnails($rrd,\%dir,$hostname) if defined $opt{t};
108     create_graphs($rrd,\%dir,$hostname) if defined $opt{g};
109     }
110    
111     exit;
112    
113    
114    
115    
116     sub create_graphs {
117     my ($rrd,$dir,$hostname,@options) = @_;
118    
119     my ($caller) = ((caller(1))[3] || '') =~ /.*::(.+)$/;
120     my $thumbnails = defined $caller && $caller eq 'create_thumbnails' ? 1 : 0;
121     my $destdir = $thumbnails ? $dir->{thumbnails} : $dir->{graphs};
122    
123     my @colour_theme = (color => [ THEME ]);
124     my $gdefs = read_graph_data("$dir->{etc}/graph.defs");
125     my @hosts = defined $hostname ? ($hostname)
126     : grep { -d catdir($dir->{data}, $_) } list_dir("$dir->{data}");
127    
128     # For each hostname
129     for my $hostname (sort @hosts) {
130     # Create the graph directory for this hostname
131     my $destination = "$destdir/$hostname";
132     File::Path::mkpath($destination) unless -d $destination;
133    
134     # For each RRD
135     for my $file (grep { $_ =~ /\.rrd$/i && !-d catfile($dir->{data},$hostname,$_) }
136     list_dir(catdir($dir->{data},$hostname))
137     ) {
138    
139     # next unless $file =~ /cpu_utilisation/;
140    
141     my $rrdfile = catfile($dir->{data},$hostname,$file);
142     my $graph = basename($file,'.rrd');
143     my $gdef = graph_def($gdefs,$graph);
144    
145     # Make sure we parse these raw commands with care
146     my @raw_cmd_list = qw(DEF CDEF VDEF TEXTALIGN AREA STACK LINE\d* HRULE\d* VRULE\d* TICK SHIFT GPRINT PRINT COMMENT);
147     my $raw_cmd_regex = '('.join('|',@raw_cmd_list).')';
148     # my $raw_cmd_regex = qr/^(?:[VC]?DEF|G?PRINT|COMMENT|[HV]RULE\d*|LINE\d*|AREA|TICK|SHIFT|STACK|TEXTALIGN)$/i;
149     my @raw_commands;
150     my @def_sources;
151     my @def_sources_draw;
152    
153     # Allow users to put raw commands in the graph.defs file
154     for my $raw_cmd (@raw_cmd_list) {
155     for my $cmd (grep(/^$raw_cmd$/i, keys %{$gdef})) {
156     my $values = $gdef->{$cmd};
157     $values = [($values)] unless ref($values);
158     for my $v (@{$values}) {
159     push @raw_commands, (sprintf('%s:%s', uc($cmd), $v) => '');
160     if ($cmd =~ /^[CV]?DEF$/i && $v =~ /^([a-z0-9\_\-]{1,30})=/) {
161     push @def_sources, $1;
162     } elsif ($cmd =~ /^(?:LINE\d*|AREA|G?PRINT|TICK|STACK)$/i && $v =~ /^([a-z0-9\_\-]{1,30})[#:]/) {
163     push @def_sources_draw, $1;
164     }
165     }
166     }
167     }
168    
169     # Wrap the RRD::Simple calls in an eval() block just in case
170     # the explode in a big nasty smelly heap!
171     eval {
172    
173     # Anything that doesn't start with ^source(?:s|_) should just
174     # be pushed on to the RRD::Simple->graph option stack (So this
175     # would NOT include the "sources" option).
176     my @graph_opts = map { ($_ => $gdef->{$_}) }
177     grep(!/^source(s|_)/ && !/^$raw_cmd_regex$/i, keys %{$gdef});
178    
179     # Anything that starts with ^source_ should be split up and passed
180     # as a hash reference in to the RRD::Simple->graph option stack
181     # (This would NOT include the "sources" option).
182     push @graph_opts, map {
183     # If we see a value from a key/value pair that looks
184     # like it might be quoted and comma seperated,
185     # "like this", 'then we should','split especially'
186     if ($gdef->{$_} =~ /["']\s*,\s*["']/) {
187     ($_ => [ split(/\s*["']\s*,\s*["']\s*/,$gdef->{$_}) ])
188    
189     # Otherwise just split on whitespace like the old
190     # version of rrd-server.pl used to do.
191     } else {
192     ($_ => [ split(/\s+/,$gdef->{$_}) ])
193     }
194     } grep(/^source_/,keys %{$gdef});
195    
196     # By default we want to tell RRDtool to be lazy and only generate
197     # graphs when it's actually necessary. If we have the -f for force
198     # flag then we won't let RRDtool be economical.
199     push @graph_opts, ('lazy','') unless exists $opt{f};
200    
201     # Only draw the sources we've been told to, and only
202     # those that actually exist in the RRD file
203     my @rrd_sources = $rrd->sources($rrdfile);
204     if (defined $gdef->{sources}) {
205     my @sources;
206     for my $ds (split(/(?:\s+|\s*,\s*)/,$gdef->{sources})) {
207     push @sources, $ds if grep(/^$ds$/,@rrd_sources);
208     }
209     push @graph_opts, ('sources',\@sources);
210     } elsif (!@def_sources && !@def_sources_draw) {
211     push @graph_opts, ('sources', [ sort @rrd_sources ]);
212     } else {
213     push @graph_opts, ('sources', undef);
214     }
215    
216     printf "Generating %s/%s/%s ...\n",
217     $hostname,
218     ($thumbnails ? 'thumbnails' : 'graphs'),
219     $graph if $opt{V};
220    
221     # Generate the graph and capture the results to
222     # write the text file output in the same directory
223     my @stack = ($rrdfile);
224     push @stack, @raw_commands if @raw_commands;
225     push @stack, ( destination => $destination );
226     push @stack, ( timestamp => 'both' );
227     push @stack, @colour_theme if @colour_theme;
228     push @stack, @options if @options;
229     push @stack, @graph_opts if @graph_opts;
230     write_txt($rrd->graph(@stack));
231    
232     my $glob = catfile($destination,"$graph*.png");
233     my @images = glob($glob);
234     warn "[Warning] $rrdfile: Looks like \$rrd->graph() failed to generate any images in '$glob'\n."
235     unless @images;
236     };
237     warn "[Warning] $rrdfile: => $@" if $@;
238     }
239     }
240     }
241    
242     sub graph_def {
243     my ($gdefs,$graph) = @_;
244    
245     my $rtn = {};
246     for (keys %{$gdefs->{graph}}) {
247     my $graph_key = qr(^$_$);
248     if (my ($var) = $graph =~ /$graph_key/) {
249     $rtn = { %{$gdefs->{graph}->{$_}} };
250     unless (defined $var && "$var" ne "1") {
251     ($var) = $graph =~ /_([^_]+)$/;
252     }
253     for my $key (keys %{$rtn}) {
254     $rtn->{$key} =~ s/\$1/$var/g;
255     }
256     last;
257     }
258     }
259    
260     return $rtn;
261     }
262    
263     sub list_dir {
264     my $dir = shift;
265     my @items = ();
266     opendir(DH,$dir) || die "Unable to open file handle for directory '$dir': $!";
267     @items = grep(!/^\./,readdir(DH));
268     closedir(DH) || die "Unable to close file handle for directory '$dir': $!";
269     return @items;
270     }
271    
272     sub create_thumbnails {
273     my ($rrd,$dir,$hostname) = @_;
274     my @thumbnail_options = (only_graph => '', width => 125, height => 32);
275     create_graphs($rrd,$dir,$hostname,@thumbnail_options);
276     }
277    
278     sub update_rrd {
279     my ($rrd,$dir,$hostname) = @_;
280     my $filename = shift @ARGV || undef;
281    
282     # Check out the input data
283     die "Input data file '$filename' does not exist.\n"
284     if defined $filename && !-f $filename;
285     die "No data recieved while expecting STDIN data from rrd-client.pl.\n"
286     if !$filename && !key_ready();
287    
288     # Check the hostname is sane
289     die "Hostname '$hostname' contains disallowed characters.\n"
290     if $hostname =~ /[^\w\-\.\d]/ || $hostname =~ /^\.|\.$/;
291    
292     # Create the data directory for the RRD file if it doesn't exist
293     File::Path::mkpath(catdir($dir->{data},$hostname)) unless -d catdir($dir->{data},$hostname);
294    
295     # Open the input file if specified
296     if (defined $filename) {
297     open(FH,'<',$filename) || die "[Error] $rrd: Unable to open file handle for file '$filename': $!";
298     select FH;
299     };
300    
301     # Parse the data
302     my %data = ();
303     while (local $_ = <>) {
304     my ($path,$value) = split(/\s+/,$_);
305     my ($time,@path) = split(/\./,$path);
306     my $key = pop @path;
307    
308     # Check that none of the data is bogus or bollocks
309     my $bogus = 0;
310     $bogus++ unless $time =~ /^\d+$/;
311     $bogus++ unless $value =~ /^[\d\.]+$/;
312     for (@path) {
313     $bogus++ unless /^[\w\-\_\.\d]+$/;
314     }
315     next if $bogus;
316    
317     my $rrdfile = catfile($dir->{data},$hostname,join('_',@path).'.rrd');
318     $data{$rrdfile}->{$time}->{$key} = $value;
319     }
320    
321     # Process the data
322     for my $rrdfile (sort keys %data) {
323     for my $time (sort keys %{$data{$rrdfile}}) {
324     eval {
325     create_rrd($rrd,$dir,$rrdfile,$data{$rrdfile}->{$time})
326     unless -f $rrdfile;
327     $rrd->update($rrdfile, $time, %{$data{$rrdfile}->{$time}});
328     };
329     warn "[Warning] $rrdfile: $@" if $@;
330     }
331     }
332    
333     # Close the input file if specified
334     if (defined $filename) {
335     select STDOUT;
336     close(FH) || warn "[Warning] $rrd: Unable to close file handle for file '$filename': $!";
337     }
338    
339     return $hostname;
340     }
341    
342     sub create_rrd {
343     my ($rrd,$dir,$rrdfile,$data) = @_;
344     my $defs = read_create_data(catfile($dir->{etc},'create.defs'));
345    
346     # Figure out what DS types to use
347     my %create = map { ($_ => 'GAUGE') } sort keys %{$data};
348     while (my ($match,$def) = each %{$defs}) {
349     next unless basename($rrdfile,qw(.rrd)) =~ /$match/;
350     for my $ds (keys %create) {
351     $create{$ds} = $def->{'*'}->{type} if defined $def->{'*'}->{type};
352     $create{$ds} = $def->{lc($ds)}->{type} if defined $def->{lc($ds)}->{type};
353     }
354     }
355    
356     # Create the RRD file
357     $rrd->create($rrdfile, %create);
358    
359     # Tune to use min and max values if specified
360     while (my ($match,$def) = each %{$defs}) {
361     next unless basename($rrdfile,qw(.rrd)) =~ /$match/;
362     for my $ds ($rrd->sources($rrdfile)) {
363     my $min = defined $def->{lc($ds)}->{min} ? $def->{lc($ds)}->{min} :
364     defined $def->{'*'}->{min} ? $def->{'*'}->{min} : undef;
365     RRDs::tune($rrdfile,'-i',"$ds:$min") if defined $min;
366    
367     my $max = defined $def->{lc($ds)}->{max} ? $def->{lc($ds)}->{max} :
368     defined $def->{'*'}->{max} ? $def->{'*'}->{max} : undef;
369     RRDs::tune($rrdfile,'-a',"$ds:$max") if defined $max;
370     }
371     }
372     }
373    
374     sub HELP_MESSAGE {
375     print qq{Syntax: rrd-server.pl <-u hostname,-g,-t,-V|-h|-v> [inputfile]
376     -u <hostname> Update RRD data for <hostname>
377     -g Create graphs from RRD data
378     -t Create thumbnails from RRD data
379     -V Display verbose progress information
380     -v Display version information
381     -h Display this help\n};
382     }
383    
384     # Display version
385     sub VERSION { &VERSION_MESSAGE; }
386     sub VERSION_MESSAGE {
387     print "$0 version $VERSION ".'($Id: rrd-server.pl 1092 2008-01-23 14:23:51Z nicolaw $)'."\n";
388     }
389    
390     sub key_ready {
391     my ($rin, $nfd) = ('','');
392     vec($rin, fileno(STDIN), 1) = 1;
393     return $nfd = select($rin,undef,undef,3);
394     }
395    
396     sub read_graph_data {
397     my $filename = shift || undef;
398    
399     my %config = ();
400     eval {
401     my $conf = new Config::General(
402     -ConfigFile => $filename,
403     -LowerCaseNames => 1,
404     -UseApacheInclude => 1,
405     -IncludeRelative => 1,
406     -MergeDuplicateBlocks => 1,
407     -AllowMultiOptions => 1,
408     -AutoTrue => 1,
409     );
410     %config = $conf->getall;
411     };
412     warn "[Warning] $@" if $@;
413    
414     return \%config;
415     }
416    
417     sub read_create_data {
418     my $filename = shift || undef;
419     my %defs = ();
420    
421     # Open the input file if specified
422     my @data;
423     if (defined $filename && -f $filename) {
424     open(FH,'<',$filename) || die "Unable to open file handle for file '$filename': $!";
425     @data = <FH>;
426     close(FH) || warn "Unable to close file handle for file '$filename': $!";
427     } else {
428     @data = <DATA>;
429     }
430    
431     # Parse the file that you've just selected
432     for (@data) {
433     last if /^__END__\s*$/;
434     next if /^\s*$/ || /^\s*#/;
435    
436     my %def = ();
437     @def{qw(rrdfile ds type min max)} = split(/\s+/,$_);
438     next unless defined $def{ds};
439     $def{ds} = lc($def{ds});
440     $def{rrdfile} = qr($def{rrdfile});
441     for (keys %def) {
442     if (!defined $def{$_} || $def{$_} eq '-') {
443     delete $def{$_};
444     } elsif ($_ =~ /^(min|max)$/ && $def{$_} !~ /^[\d\.]+$/) {
445     delete $def{$_};
446     } elsif ($_ eq 'type' && $def{$_} !~ /^(GAUGE|COUNTER|DERIVE|ABSOLUTE|COMPUTE)$/i) {
447     delete $def{$_};
448     }
449     }
450    
451     $defs{$def{rrdfile}}->{$def{ds}} = {
452     map { ($_ => $def{$_}) } grep(!/^(rrdfile|ds)$/,keys %def)
453     };
454     }
455    
456     return \%defs;
457     }
458    
459    
460    
461    
462     ##
463     ## This processing and robustness of this routine is pretty
464     ## bloody dire and awful. It needs to be rewritten with crap
465     ## input data in mind rather than patching it every time I
466     ## find a new scenario for the data to not be as expected!! ;-)
467     ##
468    
469     sub write_txt {
470     my %rtn = @_;
471     while (my ($period,$data) = each %rtn) {
472     my $filename = shift @{$data};
473     last if $filename =~ m,/thumbnails/,;
474    
475     my %values = ();
476     my $max_len = 0;
477     for (@{$data->[0]}) {
478     my ($ds,$k,$v) = split(/\s+/,$_);
479     next unless defined($ds) && length($ds) && defined($k);
480     $values{$ds}->{$k} = $v;
481     $max_len = length($ds) if length($ds) > $max_len;
482     }
483    
484     if (open(FH,'>',"$filename.txt")) {
485     printf FH "%s (%dx%d) %dK\n\n",
486     basename($filename),
487     (defined($data->[1]) ? $data->[1] : -1),
488     (defined($data->[2]) ? $data->[2] : -1),
489     (-e $filename ? (stat($filename))[7]/1024 : 0);
490    
491     for my $ds (sort keys %values) {
492     for (qw(min max last)) {
493     $values{$ds}->{$_} = ''
494     unless defined $values{$ds}->{$_};
495     }
496     printf FH "%-${max_len}s min: %s, max: %s, last: %s\n", $ds,
497     $values{$ds}->{min}, $values{$ds}->{max}, $values{$ds}->{last};
498     }
499     close(FH);
500     }
501     }
502     }
503    
504    
505    
506    
507     1;
508    
509    
510     __DATA__
511    
512     # * means all
513     # - means undef/na
514    
515     # rrdfile ds type min max
516     ^net_traffic_.+ Transmit DERIVE 0 -
517     ^net_traffic_.+ Receive DERIVE 0 -
518    
519     # 10000000000 = 10 gigabit
520     # 1000000000 = 1 gigabit
521     # 100000000 = 100 megabit
522     # rrdfile ds type min max
523     ^switch_traffic$ ifInOctets COUNTER 0 10000000000
524     ^switch_traffic$ ifOutOctets COUNTER 0 10000000000
525     ^switch_traffic_port\d+$ ifInOctets COUNTER 0 1000000000
526     ^switch_traffic_port\d+$ ifOutOctets COUNTER 0 1000000000
527    
528     # rrdfile ds type min max
529     ^hw_irq_interrupts_cpu\d+$ * DERIVE 0 -
530    
531     # rrdfile ds type min max
532     ^hdd_io_.+ * DERIVE 0 -
533    
534     # rrdfile ds type min max
535     ^net_nfs_operations$ * DERIVE 0 -
536    
537     # rrdfile ds type min max
538     ^apache_status$ ReqPerSec DERIVE 0 -
539     ^apache_status$ BytesPerSec DERIVE 0 -
540     ^apache_logs$ * DERIVE 0 -
541    
542     # rrdfile ds type min max
543     ^db_mysql_activity$ * DERIVE 0 -
544     ^db_mysql_activity_com$ * DERIVE 0 -
545     ^db_mysql_activity_scan$ * DERIVE 0 -
546     ^db_mysql_threads$ Threads_created DERIVE 0 -
547     ^db_mysql_traffic$ * DERIVE 0 -
548     ^db_mysql_connections$ * DERIVE 0 -
549     ^db_mysql_created_tmp$ * DERIVE 0 -
550    
551     # rrdfile ds type min max
552     ^mail_postfix_traffic$ * ABSOLUTE 0 -
553    
554     __END__
555    
556    

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26