--- trunk/bin/irc-logger.pl 2008/03/06 18:03:05 84 +++ trunk/bin/irc-logger.pl 2008/03/06 22:16:27 85 @@ -63,14 +63,16 @@ # number of last tags to keep in circular buffer my $last_x_tags = 50; +# don't pull rss feeds more often than this +my $rss_min_delay = 60; +$rss_min_delay = 15; + my $http_port = $NICK =~ m/-dev/ ? 8001 : 8000; my $url = "http://$HOSTNAME:$http_port"; ## END CONFIG - - use POE qw(Component::IRC Wheel::FollowTail Component::Server::HTTP); use HTTP::Status; use DBI; @@ -174,7 +176,7 @@ my $dbh = DBI->connect($DSN,"","", { RaiseError => 1, AutoCommit => 1 }) || die $DBI::errstr; my $sql_schema = { - log => ' + log => qq{ create table log ( id serial, time timestamp default now(), @@ -188,17 +190,31 @@ create index log_time on log(time); create index log_channel on log(channel); create index log_nick on log(nick); - ', - meta => ' + }, + meta => q{ create table meta ( nick text not null, channel text not null, name text not null, value text, - changed timestamp default now(), + changed timestamp default 'now()', primary key(nick,channel,name) ); - ', + }, + feeds => qq{ +create table feeds ( + id serial, + url text not null, + name text, + delay interval not null default '30 sec', --'5 min', + active boolean default true, + last_update timestamp default 'now()', + polls int default 0, + updates int default 0 +); +create unique index feeds_url on feeds(url); +insert into feeds (url,name) values ('http://wiki.razmjenavjestina.org/feed/workspace/razmjenavjestina?category=Recent%20Changes','wiki'); + }, }; foreach my $table ( keys %$sql_schema ) { @@ -624,6 +640,94 @@ exit; } +# +# RSS follow +# + +my $_rss; + + +sub rss_fetch { + my ($args) = @_; + + # how many messages to send out when feed is seen for the first time? + my $send_rss_msgs = 1; + + my $feed = XML::Feed->parse(URI->new( $args->{url} )); + if ( ! $feed ) { + _log("can't fetch RSS ", $args->{url}); + return; + } + my $updates = 0; + for my $entry ($feed->entries) { + + # seen allready? + return if $_rss->{$feed->link}->{seen}->{$entry->id}++ > 0; + + sub prefix { + my ($txt,$var) = @_; + $var =~ s/^\s+//g; + return $txt . $var if $var; + } + + my $msg; + $msg .= prefix( 'From: ' , $feed->title ); + $msg .= prefix( ' by ' , $entry->author ); + $msg .= prefix( ' -- ' , $entry->link ); +# $msg .= prefix( ' id ' , $entry->id ); + + _log('RSS', $msg); + + if ( $args->{kernel} && $send_rss_msgs ) { + warn "# sending to $CHANNEL\n"; + $send_rss_msgs--; + $args->{kernel}->post( $IRC_ALIAS => notice => $CHANNEL, $msg ); + $updates++; + } + } + + my $sql = qq{ update feeds set last_update = now(), polls = polls + 1 }; + $sql .= qq{, updates = updates + $updates } if $updates; + $sql .= qq{where id = } . $args->{id}; + $dbh->do( $sql ); + + return $updates; +} + +sub rss_fetch_all { + my $kernel = shift; + my $sql = qq{ + select id, url, name + from feeds + where active is true + }; + # limit to newer feeds only if we are not sending messages out + $sql .= qq{ and last_update + delay < now() } if $kernel; + my $sth = $dbh->prepare( $sql ); + $sth->execute(); + warn "# ",$sth->rows," active RSS feeds\n"; + my $count = 0; + while (my $row = $sth->fetchrow_hashref) { + warn "+++ fetch RSS feed: ",dump( $row ); + $row->{kernel} = $kernel if $kernel; + $count += rss_fetch( $row ); + } + return "OK, fetched $count posts from " . $sth->rows . " feeds"; +} + +my $rss_last_poll = time(); + +sub rss_check_updates { + my $kernel = shift; + my $t = time(); + if ( $rss_last_poll - $t > $rss_min_delay ) { + $rss_last_poll = $t; + _log rss_fetch_all( $kernel ); + } +} + +# seed rss seen cache so we won't send out all items on startup +_log rss_fetch_all; # # POE handing part @@ -635,8 +739,8 @@ POE::Component::IRC->new($IRC_ALIAS); -POE::Session->create( inline_states => - {_start => sub { +POE::Session->create( inline_states => { + _start => sub { $_[KERNEL]->post($IRC_ALIAS => register => 'all'); $_[KERNEL]->post($IRC_ALIAS => connect => $CONNECT); }, @@ -677,13 +781,14 @@ irc_ping => sub { _log( "pong ", $_[ARG0] ); $ping->{ $_[ARG0] }++; + rss_check_updates( $_[KERNEL] ); }, irc_invite => sub { my $kernel = $_[KERNEL]; my $nick = (split /!/, $_[ARG0])[0]; my $channel = $_[ARG1]; - warn "invited to $channel by $nick"; + _log "invited to $channel by $nick"; $_[KERNEL]->post( $IRC_ALIAS => privmsg => $nick, "how nice of you to invite me to $channel, I'll be right there..." ); $_[KERNEL]->post($IRC_ALIAS => join => $channel); @@ -823,6 +928,26 @@ $res = "config option $op doesn't exist"; } } + } elsif ($msg =~ m/^rss-update/) { + $res = rss_fetch_all( $_[KERNEL] ); + } elsif ($msg =~ m/^rss-clean/) { + $_rss = undef; + $res = "OK, cleaned RSS cache"; + } elsif ($msg =~ m!^rss-(add|remove|stop|start)\s+(http://\S+)\s*(.*)!) { + my $sql = { + add => qq{ insert into feeds (url,name) values (?,?) }, +# remove => qq{ delete from feeds where url = ? and name = ? }, + start => qq{ update feeds set active = true where url = ? -- ? }, + stop => qq{ update feeds set active = false where url = ? -- ? }, + + }; + if (my $q = $sql->{$1} ) { + my $sth = $dbh->prepare( $q ); + warn "## SQL $q ( $2 | $3 )\n"; + eval { $sth->execute( $2, $3 ) }; + } + + $res ||= "OK, RSS $1 : $2 - $3"; } if ($res) { @@ -831,6 +956,7 @@ $_[KERNEL]->post( $IRC_ALIAS => privmsg => $nick, $res ); } + rss_check_updates( $_[KERNEL] ); }, irc_477 => sub { _log "# irc_477: ",$_[ARG1]; @@ -1004,7 +1130,7 @@ my $search = $q->param('search') || $q->param('grep') || ''; - if ($request->url =~ m#/rss(?:/(tags|last-tag)\w*(?:=(\d+))?)?#i) { + if ($request->url =~ m#/rss(?:/(tags|last-tag|follow.*)\w*(?:=(\d+))?)?#i) { my $show = lc($1); my $nr = $2; @@ -1016,6 +1142,7 @@ #warn "create $type feed from ",dump( @last_tags ); my $feed = XML::Feed->new( $type ); + $feed->link( $url ); if ( $show eq 'tags' ) { $nr ||= 50; @@ -1042,7 +1169,6 @@ $nr = $last_x_tags if $nr > $last_x_tags; $feed->title( "last $nr tagged messages from $CHANNEL" ); - $feed->link( $url ); $feed->description( "collects messages which have tags// in them" ); foreach my $m ( @last_tags ) { @@ -1071,8 +1197,25 @@ } + } elsif ( $show =~ m/^follow/ ) { + + $feed->title( "Feeds which this bot follows" ); + + my $sth = $dbh->prepare( qq{ select * from feeds order by last_update desc } ); + $sth->execute; + while (my $row = $sth->fetchrow_hashref) { + my $feed_entry = XML::Feed::Entry->new($type); + $feed_entry->title( $row->{name} ); + $feed_entry->link( $row->{url} ); + $feed_entry->issued( DateTime::Format::Flexible->build( $row->{last_update} ) ); + $feed_entry->content( + '' . dump( $row ) . ']]>' + ); + $feed->add_entry( $feed_entry ); + } + } else { - warn "!! unknown rss request for $show\n"; + _log "unknown rss request ",$request->url; return RC_DENY; }