--- trunk/bin/irc-logger.pl 2007/03/18 16:45:18 52
+++ trunk/bin/irc-logger.pl 2008/02/20 20:26:45 79
@@ -32,7 +32,8 @@
## CONFIG
-my $HOSTNAME = `hostname`;
+my $HOSTNAME = `hostname -f`;
+chomp($HOSTNAME);
my $NICK = 'irc-logger';
$NICK .= '-dev' if ($HOSTNAME =~ m/llin/);
@@ -58,6 +59,13 @@
my $sleep_on_error = 5;
+# number of last tags to keep in circular buffer
+my $last_x_tags = 50;
+
+my $http_port = $NICK =~ m/-dev/ ? 8001 : 8000;
+
+my $url = "http://$HOSTNAME:$http_port";
+
## END CONFIG
@@ -73,7 +81,16 @@
use HTML::CalendarMonthSimple;
use Getopt::Long;
use DateTime;
+use URI::Escape;
use Data::Dump qw/dump/;
+use DateTime::Format::ISO8601;
+use Carp qw/confess/;
+use XML::Feed;
+use DateTime::Format::Flexible;
+
+my $use_twitter = 1;
+eval { require Net::Twitter; };
+$use_twitter = 0 if ($@);
my $import_dircproxy;
my $log_path;
@@ -82,12 +99,58 @@
'log:s' => \$log_path,
);
+$SIG{__DIE__} = sub {
+ confess "fatal error";
+};
+
open(STDOUT, '>', $log_path) || warn "can't redirect log to $log_path: $!";
sub _log {
print strftime($TIMESTAMP,localtime()), ' ', join(" ",@_), $/;
}
+# HTML formatters
+
+my %escape = ('<'=>'<', '>'=>'>', '&'=>'&', '"'=>'"');
+my $escape_re = join '|' => keys %escape;
+
+my $tag_regex = '\b([\w-_]+)//';
+
+my %nick_enumerator;
+my $max_color = 0;
+
+my $filter = {
+ message => sub {
+ my $m = shift || return;
+
+ # protect HTML from wiki modifications
+ sub e {
+ my $t = shift;
+ return 'uri_unescape{' . uri_escape($t) . '}';
+ }
+
+ $m =~ s/($escape_re)/$escape{$1}/gs;
+ $m =~ s#($RE{URI}{HTTP})#e(qq{$1})#egs ||
+ $m =~ s#\/(\w+)\/#$1#gs;
+ $m =~ s#$tag_regex#e(qq{$1})#egs;
+ $m =~ s#\*(\w+)\*#$1#gs;
+ $m =~ s#_(\w+)_#$1#gs;
+
+ $m =~ s#uri_unescape{([^}]+)}#uri_unescape($1)#egs;
+ return $m;
+ },
+ nick => sub {
+ my $n = shift || return;
+ if (! $nick_enumerator{$n}) {
+ my $max = scalar keys %nick_enumerator;
+ $nick_enumerator{$n} = $max + 1;
+ }
+ return '' . $n . '';
+ },
+};
+
my $dbh = DBI->connect($DSN,"","", { RaiseError => 1, AutoCommit => 1 }) || die $DBI::errstr;
my $sql_schema = {
@@ -187,7 +250,6 @@
my $tags;
-my $tag_regex = '\b([\w-_]+)//';
=head2 get_from_log
@@ -224,14 +286,16 @@
sub get_from_log {
my $args = {@_};
- $args->{fmt} ||= {
- date => '[%s] ',
- time => '{%s} ',
- time_channel => '{%s %s} ',
- nick => '%s: ',
- me_nick => '***%s ',
- message => '%s',
- };
+ if ( ! $args->{fmt} ) {
+ $args->{fmt} = {
+ date => '[%s] ',
+ time => '{%s} ',
+ time_channel => '{%s %s} ',
+ nick => '%s: ',
+ me_nick => '***%s ',
+ message => '%s',
+ };
+ }
my $sql_message = qq{
select
@@ -254,27 +318,50 @@
my $sql = $context ? $sql_context : $sql_message;
- $sql .= " where message ilike ? or nick ilike ? " if ($args->{search});
- $sql .= " where id in (" . join(",", @{ $tags->{ $args->{tag} } }) . ") " if ($args->{tag} && $tags->{ $args->{tag} });
- $sql .= " where date(time) = ? " if ($args->{date});
- $sql .= " order by log.time desc";
- $sql .= " limit " . $args->{limit} if ($args->{limit});
+ sub check_date {
+ my $date = shift || return;
+ my $new_date = eval { DateTime::Format::ISO8601->parse_datetime( $date )->ymd; };
+ if ( $@ ) {
+ warn "invalid date $date\n";
+ $new_date = DateTime->now->ymd;
+ }
+ return $new_date;
+ }
+
+ my @where;
+ my @args;
- my $sth = $dbh->prepare( $sql );
if (my $search = $args->{search}) {
$search =~ s/^\s+//;
$search =~ s/\s+$//;
- $sth->execute( ( '%' . $search . '%' ) x 2 );
- _log "search for '$search' returned ", $sth->rows, " results ", $context || '';
- } elsif (my $tag = $args->{tag}) {
- $sth->execute();
- _log "tag '$tag' returned ", $sth->rows, " results ", $context || '';
- } elsif (my $date = $args->{date}) {
- $sth->execute($date);
- _log "found ", $sth->rows, " messages for date $date ", $context || '';
- } else {
- $sth->execute();
+ push @where, 'message ilike ? or nick ilike ?';
+ push @args, ( ( '%' . $search . '%' ) x 2 );
+ _log "search for '$search'";
+ }
+
+ if ($args->{tag} && $tags->{ $args->{tag} }) {
+ push @where, 'id in (' . join(',', @{ $tags->{ $args->{tag} } }) . ')';
+ _log "search for tags $args->{tag}";
}
+
+ if (my $date = $args->{date} ) {
+ $date = check_date( $date );
+ push @where, 'date(time) = ?';
+ push @args, $date;
+ _log "search for date $date";
+ }
+
+ $sql .= " where " . join(" and ", @where) if @where;
+
+ $sql .= " order by log.time desc";
+ $sql .= " limit " . $args->{limit} if ($args->{limit});
+
+ #warn "### sql: $sql ", dump( @args );
+
+ my $sth = $dbh->prepare( $sql );
+ eval { $sth->execute( @args ) };
+ return if $@;
+
my $last_row = {
date => '',
time => '',
@@ -395,10 +482,12 @@
=head2 add_tag
- add_tag( id => 42, message => 'irc message' );
+ add_tag( id => 42, message => 'irc message', nick => 'foobar' [, me => 1 ] );
=cut
+my @last_tags;
+
sub add_tag {
my $arg = {@_};
@@ -407,13 +496,23 @@
my $m = $arg->{message};
from_to('UTF-8', 'iso-8859-2', $m) if (is_utf8($m));
+ my @tags;
+
while ($m =~ s#$tag_regex##s) {
my $tag = $1;
next if (! $tag || $tag =~ m/https?:/i);
push @{ $tags->{$tag} }, $arg->{id};
#warn "+tag $tag: $arg->{id}\n";
- $cloud->add($tag, "?tag=$tag", scalar @{$tags->{$tag}} + 1);
+ $cloud->add($tag, "$url?tag=$tag", scalar @{$tags->{$tag}} + 1);
+ push @tags, $tag;
+
+ }
+
+ if ( @tags ) {
+ pop @last_tags if $#last_tags == $last_x_tags;
+ unshift @last_tags, { tags => [ @tags ], %$arg };
}
+
}
=head2 seed_tags
@@ -423,14 +522,14 @@
=cut
sub seed_tags {
- my $sth = $dbh->prepare(qq{ select id,message from log where message like '%//%' });
+ my $sth = $dbh->prepare(qq{ select id,message,nick,me,time from log where message like '%//%' order by time asc });
$sth->execute;
while (my $row = $sth->fetchrow_hashref) {
add_tag( %$row );
}
foreach my $tag (keys %$tags) {
- $cloud->add($tag, "?tag=$tag", scalar @{$tags->{$tag}} + 1);
+ $cloud->add($tag, "$url?tag=$tag", scalar @{$tags->{$tag}} + 1);
}
}
@@ -443,7 +542,7 @@
channel => '#foobar',
me => 0,
nick => 'dpavlin',
- msg => 'test message',
+ message => 'test message',
time => '2006-06-25 18:57:18',
);
@@ -455,26 +554,26 @@
sub save_message {
my $a = {@_};
+ confess "have msg" if $a->{msg};
$a->{me} ||= 0;
$a->{time} ||= strftime($TIMESTAMP,localtime());
_log
$a->{channel}, " ",
$a->{me} ? "***" . $a->{nick} : "<" . $a->{nick} . ">",
- " " . $a->{msg};
+ " " . $a->{message};
- from_to($a->{msg}, 'UTF-8', $ENCODING);
+ from_to($a->{message}, 'UTF-8', $ENCODING);
- $sth->execute($a->{channel}, $a->{me}, $a->{nick}, $a->{msg}, $a->{time});
- add_tag( id => $dbh->last_insert_id(undef,undef,"log",undef),
- message => $a->{msg});
+ $sth->execute($a->{channel}, $a->{me}, $a->{nick}, $a->{message}, $a->{time});
+ add_tag( id => $dbh->last_insert_id(undef,undef,"log",undef), %$a );
}
if ($import_dircproxy) {
open(my $l, $import_dircproxy) || die "can't open $import_dircproxy: $!";
warn "importing $import_dircproxy...\n";
- my $tz_offset = 2 * 60 * 60; # TZ GMT+2
+ my $tz_offset = 1 * 60 * 60; # TZ GMT+2
while(<$l>) {
chomp;
if (/^@(\d+)\s(\S+)\s(.+)$/) {
@@ -492,7 +591,7 @@
channel => $CHANNEL,
me => $me,
nick => $nick,
- msg => $msg,
+ message => $msg,
time => $dt->ymd . " " . $dt->hms,
) if ($nick !~ m/^-/);
@@ -534,7 +633,7 @@
my $channel = $_[ARG1]->[0];
my $msg = $_[ARG2];
- save_message( channel => $channel, me => 0, nick => $nick, msg => $msg);
+ save_message( channel => $channel, me => 0, nick => $nick, message => $msg);
meta( $nick, $channel, 'last-msg', $msg );
},
irc_ctcp_action => sub {
@@ -543,10 +642,15 @@
my $channel = $_[ARG1]->[0];
my $msg = $_[ARG2];
- save_message( channel => $channel, me => 1, nick => $nick, msg => $msg);
+ save_message( channel => $channel, me => 1, nick => $nick, message => $msg);
- if ( my $twitter = ( $nick, $channel, 'twitter' ) ) {
- _log("FIXME: send twitter for $nick on $channel [$twitter]");
+ if ( $use_twitter ) {
+ if ( my $twitter = meta( $nick, $NICK, 'twitter' ) ) {
+ my ($login,$passwd) = split(/\s+/,$twitter,2);
+ _log("sending twitter for $nick/$login on $channel ");
+ my $bot = Net::Twitter->new( username=>$login, password=>$passwd );
+ $bot->update("<${channel}> $msg");
+ }
}
},
@@ -593,11 +697,11 @@
my $sth = $dbh->prepare(qq{
select
- nick,
+ trim(both '_' from nick) as nick,
count(*) as count,
sum(length(message)) as len
from log
- group by nick
+ group by trim(both '_' from nick)
order by len desc,count desc
limit $nr
});
@@ -811,14 +915,11 @@
# http server
my $httpd = POE::Component::Server::HTTP->new(
- Port => $NICK =~ m/-dev/ ? 8001 : 8000,
+ Port => $http_port,
ContentHandler => { '/' => \&root_handler },
Headers => { Server => 'irc-logger' },
);
-my %escape = ('<'=>'<', '>'=>'>', '&'=>'&', '"'=>'"');
-my $escape_re = join '|' => keys %escape;
-
my $style = <<'_END_OF_STYLE_';
p { margin: 0; padding: 0.1em; }
.time, .channel { color: #808080; font-size: 60%; }
@@ -826,24 +927,41 @@
.nick { color: #000000; font-size: 80%; padding: 2px; font-family: courier, courier new, monospace ; }
.message { color: #000000; font-size: 100%; }
.search { float: right; }
+a:link.tag, a:visited.tag { border: 1px dashed #ccc; backgound: #ccc; text-decoration: none }
+a:hover.tag { border: 1px solid #eee }
+hr { border: 1px dashed #ccc; height: 1px; clear: both; }
+/*
.col-0 { background: #ffff66 }
.col-1 { background: #a0ffff }
.col-2 { background: #99ff99 }
.col-3 { background: #ff9999 }
.col-4 { background: #ff66ff }
-a:link.tag, a:visited.tag { border: 1px dashed #ccc; backgound: #ccc; text-decoration: none }
-a:hover.tag { border: 1px solid #eee }
-hr { border: 1px dashed #ccc; height: 1px; clear: both; }
+*/
+.calendar { border: 1px solid red; width: 100%; }
+.month { border: 0px; width: 100%; }
_END_OF_STYLE_
-my $max_color = 4;
+$max_color = 0;
-my %nick_enumerator;
+my @cols = qw(
+ #ffcccc #ccffe6 #ccccff #e6ccff #ffccff #ffcce6 #ff9999 #ffcc99 #ffff99
+ #ccff99 #99ff99 #99ffcc #99ccff #9999ff #cc99ff #ff6666 #ffb366 #ffff66
+ #66ff66 #66ffb3 #66b3ff #6666ff #ff3333 #33ff33 #3399ff #3333ff #ff3399
+ #a0a0a0 #ff0000 #ffff00 #80ff00 #0000ff #8000ff #ff00ff #ff0080 #994d00
+ #999900 #009900 #cc0066 #c0c0c0 #ccff99 #99ff33 #808080 #660033 #ffffff
+);
+
+foreach my $c (@cols) {
+ $style .= ".col-${max_color} { background: $c }\n";
+ $max_color++;
+}
+warn "defined $max_color colors for users...\n";
sub root_handler {
my ($request, $response) = @_;
$response->code(RC_OK);
- $response->content_type("text/html; charset=$ENCODING");
+
+ return RC_OK if $request->uri =~ m/favicon.ico$/;
my $q;
@@ -857,54 +975,152 @@
my $search = $q->param('search') || $q->param('grep') || '';
+ if ($request->url =~ m#/rss(?:/(tags|last-tag?)\w+(?:=(\d+))?)?#i) {
+ my $show = lc($1);
+ my $nr = $2;
+
+ my $type = 'RSS'; # Atom
+
+ $response->content_type( 'application/' . lc($type) . '+xml' );
+
+ my $html = '';
+ #warn "create $type feed from ",dump( @last_tags );
+
+ my $feed = XML::Feed->new( $type );
+
+ if ( $show eq 'tags' ) {
+ $nr ||= 50;
+ $feed->title( "tags from $CHANNEL" );
+ $feed->link( "$url/tags" );
+ $feed->description( "tag cloud created from messages on channel $CHANNEL which have tags// in them" );
+ my $feed_entry = XML::Feed::Entry->new($type);
+ $feed_entry->title( "$nr tags from $CHANNEL" );
+ $feed_entry->author( $NICK );
+ $feed_entry->link( '/#tags' );
+
+ $feed_entry->content(
+ qq{}
+ . $cloud->css
+ . qq{}
+ . $cloud->html( $nr )
+ . qq{]]>}
+ );
+ $feed->add_entry( $feed_entry );
+
+ } elsif ( $show eq 'last-tag' ) {
+
+ $nr ||= $last_x_tags;
+
+ $feed->title( "last $nr tagged messages from $CHANNEL" );
+ $feed->link( $url );
+ $feed->description( "collects messages which have tags// in them" );
+
+ foreach my $m ( @last_tags ) {
+# warn dump( $m );
+ #my $tags = join(' ', @{$m->{tags}} );
+ my $feed_entry = XML::Feed::Entry->new($type);
+ $feed_entry->title( $m->{nick} . '@' . $m->{time} );
+ $feed_entry->author( $m->{nick} );
+ $feed_entry->link( '/#' . $m->{id} );
+ $feed_entry->issued( DateTime::Format::Flexible->build( $m->{time} ) );
+
+ my $message = $filter->{message}->( $m->{message} );
+ $message .= "
\n" unless $message =~ m!<(/p|br/?)>!;
+# warn "## message = $message\n";
+ from_to( $message, $ENCODING, 'UTF-8' );
+
+ #$feed_entry->summary(
+ $feed_entry->content(
+ ""
+ );
+ $feed_entry->category( join(', ', @{$m->{tags}}) );
+ $feed->add_entry( $feed_entry );
+
+ $nr--;
+ last if $nr <= 0;
+
+ }
+
+ } else {
+ warn "!! unknown rss request for $show\n";
+ return RC_DENY;
+ }
+
+ $response->content( $feed->as_xml );
+ return RC_OK;
+ }
+
+ if ( $@ ) {
+ warn "$@";
+ }
+
+ $response->content_type("text/html; charset=$ENCODING");
+
my $html =
- qq{
}; - if ($request->url =~ m#/history#) { + } + . $cloud->html(500) + . qq{
}; + + if ($request->url =~ m#/tags?#) { + # nop + } elsif ($request->url =~ m#/history#) { my $sth = $dbh->prepare(qq{ - select date(time) as date,count(*) as nr + select date(time) as date,count(*) as nr,sum(length(message)) as len from log group by date(time) order by date(time) desc }); $sth->execute(); my ($l_yyyy,$l_mm) = (0,0); + $html .= qq{
} . $cal->as_HTML() . qq{ | }; + $ord++; + $html .= qq{
} . $cal->as_HTML() . qq{ |
", get_from_log( - limit => $q->param('last') || $q->param('date') ? undef : 100, + limit => ( $q->param('last') || $q->param('date') ) ? undef : 100, search => $search || undef, tag => $q->param('tag') || undef, date => $q->param('date') || undef, fmt => { date => sub { my $date = shift || return; - qq{