--- trunk/bin/irc-logger.pl	2008/03/09 22:12:06	111
+++ trunk/bin/irc-logger.pl	2008/03/14 14:45:04	123
@@ -2,7 +2,7 @@
 use strict;
 $|++;
 
-use POE qw(Component::IRC Component::Server::HTTP);
+use POE qw(Component::IRC Component::Server::HTTP Component::Client::HTTP);
 use HTTP::Status;
 use DBI;
 use Regexp::Common qw /URI/;
@@ -18,6 +18,9 @@
 use Carp qw/confess/;
 use XML::Feed;
 use DateTime::Format::Flexible;
+use IPC::DirQueue;
+use File::Slurp;
+use Encode;
 
 =head1 NAME
 
@@ -47,6 +50,8 @@
 
 ## CONFIG
 
+my $debug = 0;
+
 my $irc_config = {
 	nick => 'irc-logger',
 	server => 'irc.freenode.net',
@@ -54,6 +59,8 @@
 	ircname => 'Anna the bot: try /msg irc-logger help',
 };
 
+my $queue_dir = './queue';
+
 my $HOSTNAME = `hostname -f`;
 chomp($HOSTNAME);
 
@@ -61,9 +68,9 @@
 my $CHANNEL = '#razmjenavjestina';
 
 if ( $HOSTNAME =~ m/llin/ ) {
-	$irc_config->{nick} = 'irc-logger-dev';
+	$irc_config->{nick} = 'irc-logger-llin';
 #	$irc_config = {
-#		nick => 'irc-logger-dev',
+#		nick => 'irc-logger-llin',
 #		server => 'localhost',
 #		port => 6668,
 #	};
@@ -75,7 +82,7 @@
 
 my @channels = ( $CHANNEL );
 
-warn "# config = ", dump( $irc_config ), $/;
+warn "## config = ", dump( $irc_config ) if $debug;
 
 my $NICK = $irc_config->{nick} or die "no nick?";
 
@@ -106,18 +113,29 @@
 GetOptions(
 	'import-dircproxy:s' => \$import_dircproxy,
 	'log:s' => \$log_path,
+	'queue:s' => \$queue_dir,
+	'debug!' => \$debug,
 );
 
 #$SIG{__DIE__} = sub {
 #	confess "fatal error";
 #};
 
-open(STDOUT, '>', $log_path) || warn "can't redirect log to $log_path: $!";
-
 sub _log {
-	print strftime($TIMESTAMP,localtime()) . ' ' . join(" ",@_) . $/;
+	print strftime($TIMESTAMP,localtime()) . ' ' . join(" ",map { ref($_) ? dump( $_ ) : $_ } @_) . $/;
+}
+
+open(STDOUT, '>', $log_path) && warn "log to $log_path: $!\n";
+
+# queue
+
+if ( ! -d $queue_dir ) {
+	warn "## creating queue directory $queue_dir";
+	mkdir $queue_dir or die "can't create queue directory $queue_dir: $!";
 }
 
+my $dq = IPC::DirQueue->new({ dir => $queue_dir });
+
 # HTML formatters
 
 my %escape = ('<'=>'&lt;', '>'=>'&gt;', '&'=>'&amp;', '"'=>'&quot;');
@@ -160,6 +178,13 @@
 	},
 };
 
+# POE IRC
+my $poe_irc = POE::Component::IRC->spawn( %$irc_config ) or 
+	die "can't start ", dump( $irc_config ), ": $!";
+
+my $irc = $poe_irc->session_id();
+_log "IRC session_id $irc";
+
 my $dbh = DBI->connect($DSN,"","", { RaiseError => 1, AutoCommit => 1 }) || die $DBI::errstr;
 $dbh->do( qq{ set client_encoding = 'UTF-8' } );
 
@@ -643,27 +668,37 @@
 
 my $_stat;
 
+POE::Component::Client::HTTP->spawn(
+	Alias	=> 'rss-fetch',
+	Timeout	=> 30,
+);
 
-sub rss_fetch {
+sub rss_parse_xml {
 	my ($args) = @_;
 
+	warn "## rss_parse_xml ",dump( @_ ) if $debug;
+
 	# how many messages to send out when feed is seen for the first time?
 	my $send_rss_msgs = 1;
 
 	_log "RSS fetch", $args->{url};
 
-	my $feed = XML::Feed->parse(URI->new( $args->{url} ));
+	my $feed = XML::Feed->parse( \$args->{xml} );
 	if ( ! $feed ) {
-		_log("can't fetch RSS ", $args->{url});
+		_log "can't fetch RSS ", $args->{url}, XML::Feed->errstr;
 		return;
 	}
 
+	$_stat->{rss}->{url2link}->{ $args->{url} } = $feed->link;
+
 	my ( $total, $updates ) = ( 0, 0 );
 	for my $entry ($feed->entries) {
 		$total++;
 
+		my $seen_times = $_stat->{rss}->{seen}->{$args->{channel}}->{$feed->link}->{$entry->id}++;
 		# seen allready?
-		next if $_stat->{rss}->{seen}->{$args->{channel}}->{$feed->link}->{$entry->id}++ > 0;
+		warn "## $seen_times ",$entry->id if $debug;
+		next if $seen_times > 0;
 
 		sub prefix {
 			my ($txt,$var) = @_;
@@ -689,18 +724,32 @@
 		$msg .= prefix( ' | ' , $entry->title );
 		$msg .= prefix( ' | ' , $link );
 #		$msg .= prefix( ' id ' , $entry->id );
+		if ( my $tags = $entry->category ) {
+			$tags =~ s!^\s+!!;
+			$tags =~ s!\s*$! !;
+			$tags =~ s!,?\s+!// !g;
+			$msg .= prefix( ' ' , $tags );
+		}
 
-		if ( $args->{kernel} && $send_rss_msgs ) {
+		if ( $seen_times == 0 && $send_rss_msgs ) {
 			$send_rss_msgs--;
 			if ( ! $args->{private} ) {
 				# FIXME bug! should be save_message
-#				save_message( channel => $args->{channel}, me => 1, nick => $NICK, message => $msg );
-				$sth_insert_log->execute( $args->{channel}, 1, $NICK, $msg, 'now()' );
+				save_message( channel => $args->{channel}, me => 1, nick => $NICK, message => $msg );
+#				$sth_insert_log->execute( $args->{channel}, 1, $NICK, $msg, 'now()' );
 			}
 			my ( $type, $to ) = ( 'notice', $args->{channel} );
 			( $type, $to ) = ( 'privmsg', $args->{nick} ) if $args->{private};
-			_log(">> $type $to |", $msg);
-			$args->{kernel}->post( irc => $type => $to, $msg );
+
+			_log("RSS generated $type to $to:", $msg);
+			# XXX enqueue message to send later
+			sub enqueue_post {
+				my $post = dump( @_ );
+				warn "## queue_post $post\n" if $debug;
+				$dq->enqueue_string( $post );
+			}
+			enqueue_post( $type => $to => $msg );
+
 			$updates++;
 		}
 	}
@@ -723,16 +772,22 @@
 		where active is true
 	};
 	# limit to newer feeds only if we are not sending messages out
-	$sql .= qq{	and last_update + delay < now() } if $kernel;
+	$sql .= qq{	and last_update + delay < now() } if defined ( $_stat->{rss}->{fetch} );
 	my $sth = $dbh->prepare( $sql );
 	$sth->execute();
 	warn "# ",$sth->rows," active RSS feeds\n";
 	my $count = 0;
 	while (my $row = $sth->fetchrow_hashref) {
-		$row->{kernel} = $kernel if $kernel;
-		$count += rss_fetch( $row );
+		warn "## queued rss-fetch for ", $row->{url} if $debug;
+		$_stat->{rss}->{fetch}->{ $row->{url} } = $row;
+		$kernel->post(
+			'rss-fetch',
+			'request',
+			'rss_response',
+			HTTP::Request->new( GET => $row->{url} ),
+		);
 	}
-	return "OK, fetched $count posts from " . $sth->rows . " feeds";
+	return "OK, scheduled " . $sth->rows . " feeds for refresh";
 }
 
 
@@ -740,26 +795,23 @@
 	my $kernel = shift;
 	$_stat->{rss}->{last_poll} ||= time();
 	my $dt = time() - $_stat->{rss}->{last_poll};
-	warn "## rss_check_updates $dt > $rss_min_delay\n";
 	if ( $dt > $rss_min_delay ) {
+		warn "## rss_check_updates $dt > $rss_min_delay\n";
 		$_stat->{rss}->{last_poll} = time();
 		_log rss_fetch_all( $kernel );
 	}
+	# XXX send queue messages
+	while ( my $job = $dq->pickup_queued_job() ) {
+		my $data = read_file( $job->get_data_path ) || die "can't load ", $job->get_data_path, ": $!";
+#		$kernel->post( $irc => $type => $to, $msg );
+		my @data = eval $data;
+		_log "IRC post from queue:", @data;
+		$kernel->post( $irc => @data );
+		$job->finish;
+		warn "## done queued job: ",dump( @data ) if $debug;
+	}
 }
 
-# seed rss seen cache so we won't send out all items on startup
-_log rss_fetch_all;
-
-#
-# POE handing part
-#
-
-my $poe_irc = POE::Component::IRC->spawn( %$irc_config ) or 
-	die "can't start ", dump( $irc_config ), ": $!";
-
-my $irc = $poe_irc->session_id();
-_log "session_id $irc";
-
 POE::Session->create( inline_states => {
 	_start => sub {      
 		$_[KERNEL]->post( $irc => register => 'all' );
@@ -770,12 +822,13 @@
 		my $poco_object = $sender->get_heap();
 		_log "connected to",$poco_object->server_name();
 		$kernel->post( $sender => join => $_ ) for @channels;
+		# seen RSS cache
+		_log rss_fetch_all( $kernel );
 		undef;
 	},
-    irc_255 => sub {	# server is done blabbing
-		$_[KERNEL]->post( $irc => join => $CHANNEL);
-		$_[KERNEL]->post( $irc => privmsg => 'nickserv', "IDENTIFY $NICK" );
-    },
+#	irc_255 => sub {	# server is done blabbing
+#		$_[KERNEL]->post( $irc => join => $CHANNEL);
+#	},
     irc_public => sub {
 		my $kernel = $_[KERNEL];
 		my $nick = (split /!/, $_[ARG0])[0];
@@ -825,6 +878,7 @@
 		my $nick = (split /!/, $_[ARG0])[0];
 		my $msg = $_[ARG2];
 		my $channel = $_[ARG1]->[0];
+		warn "# ARG = ",dump( @_[ARG0,ARG1,ARG2] ) if $debug;
 
 		my $res = "unknown command '$msg', try /msg $NICK help!";
 		my @out;
@@ -953,10 +1007,6 @@
 			}
 		} elsif ($msg =~ m/^rss-update/) {
 			$res = rss_fetch_all( $_[KERNEL] );
-		} elsif ($msg =~ m/^rss-clean/) {
-			$_stat->{rss} = undef;
-			$dbh->do( qq{ update feeds set last_update = now() - delay } );
-			$res = "OK, cleaned RSS cache";
 		} elsif ($msg =~ m/^rss-list/) {
 			my $sth = $dbh->prepare(qq{ select url,name,last_update,active,channel,nick,private from feeds });
 			$sth->execute;
@@ -964,7 +1014,7 @@
 				$_[KERNEL]->post( $irc => privmsg => $nick, join(' | ',@row) );
 			}
 			$res = '';
-		} elsif ($msg =~ m!^rss-(add|remove|stop|start)(?:-(private))?\s+(http://\S+)\s*(.*)!) {
+		} elsif ($msg =~ m!^rss-(add|remove|stop|start|clean)(?:-(private))?\s+(http://\S+)\s*(.*)!) {
 			my ( $command, $sub, $url, $arg ) = ( $1,$2,$3,$4 );
 
 			my $channel = $1 if ( $arg =~ s/\s*(#\S+)\s*// );
@@ -975,6 +1025,7 @@
 #				remove	=> qq{ delete from feeds				where url = ? and name = ? },
 				start	=> qq{ update feeds set active = true	where url = ? },
 				stop	=> qq{ update feeds set active = false	where url = ? },
+				clean	=> qq{ update feeds set last_update = now() - delay where url = ? },
 			};
 
 			if ( $command eq 'add' && ! $channel ) {
@@ -990,11 +1041,28 @@
 				if ($@) {
 					$res = "ERROR: $@";
 				} else {
-					$res = "OK, RSS [$command|$sub|$url|$arg]";
+					$res = "OK, RSS executed $command " . ( $sub ? "-$sub" : '' ) ."on $channel url $url";
+					if ( $command eq 'clean' ) {
+						my $seen = $_stat->{rss}->{seen} || die "no seen?";
+						my $want_link = $_stat->{rss}->{url2link}->{$url} || warn "no url2link($url)";
+						foreach my $c ( keys %$seen ) {
+							my $c_hash = $seen->{$c} || die "no seen->{$c}";
+							die "not HASH with rss links but ", dump($c_hash) unless ref($c_hash) eq 'HASH';
+							foreach my $link ( keys %$c_hash ) {
+								next unless $link eq $want_link;
+								_log "RSS removed seen $c $url $link";
+							}
+						}
+					}
 				}
 			} else {
 				$res = "ERROR: don't know what to do with: $msg";
 			}
+		} elsif ($msg =~ m/^rss-clean/) {
+			# this makes sense because we didn't catch rss-clean http://... before!
+			$_stat->{rss} = undef;
+			$dbh->do( qq{ update feeds set last_update = now() - delay } );
+			$res = "OK, cleaned RSS cache";
 		}
 
 		if ($res) {
@@ -1013,13 +1081,22 @@
 	irc_376 => sub {
 		_log "<< motd", $_[ARG0], "end";
 	},
+#	irc_433 => sub {
+#		print "# irc_433: ",$_[ARG1], "\n";
+#		warn "## indetify $NICK\n";
+#		$_[KERNEL]->post( $irc => privmsg => 'nickserv', "IDENTIFY $NICK" );
+#	},
+#	irc_451	# please register
 	irc_477 => sub {
 		_log "<< irc_477: ",$_[ARG1];
-		$_[KERNEL]->post( $irc => privmsg => 'nickserv', "register $NICK" );
+		_log ">> IDENTIFY $NICK";
+		$_[KERNEL]->post( $irc => privmsg => 'NickServ', "IDENTIFY $NICK" );
 	},
 	irc_505 => sub {
 		_log "<< irc_505: ",$_[ARG1];
-		$_[KERNEL]->post( $irc => privmsg => 'nickserv', "register $NICK" );
+		_log ">> register $NICK";
+		$_[KERNEL]->post( $irc => privmsg => 'NickServ', "register $NICK" );
+#		$_[KERNEL]->post( $irc => privmsg => 'nickserv', "IDENTIFY $NICK" );
 #		$_[KERNEL]->post( $irc => privmsg => 'nickserv', "set hide email on" );
 #		$_[KERNEL]->post( $irc => privmsg => 'nickserv', "set email dpavlin\@rot13.org" );
 	},
@@ -1036,20 +1113,21 @@
 		sleep($sleep_on_error);
 		$_[KERNEL]->post( $irc => connect => {} );
 	},
-#	irc_433 => sub {
-#		print "# irc_433: ",$_[ARG1], "\n";
-#		warn "## indetify $NICK\n";
-#		$_[KERNEL]->post( $irc => privmsg => 'nickserv', "IDENTIFY $NICK" );
-#	},
-#	irc_451	# please register
 	irc_notice => sub {
-		_log "<< notice",$_[ARG0];
-		if ( $_[ARG0] =~ m!/msg\s+NickServ\s+IDENTIFY!i ) {
+		_log "<< notice from ", $_[ARG0], $_[ARG1], $_[ARG2];
+		my $m = $_[ARG2];
+		if ( $m =~ m!/msg.*(NickServ).*(IDENTIFY)!i ) {
+			_log ">> suggested to $1 $2";
+			$_[KERNEL]->post( $irc => privmsg => $1, "$2 $NICK" );
+		} elsif ( $m =~ m!\Q$NICK\E.*registered!i ) {
+			_log ">> registreted, so IDENTIFY";
 			$_[KERNEL]->post( $irc => privmsg => 'nickserv', "IDENTIFY $NICK" );
+		} else {
+			warn "## ignore $m\n" if $debug;
 		}
 	},
 	irc_snotice => sub {
-		_log "<< snotice",$_[ARG0];
+		_log "<< snotice", $_[ARG0]; #dump( $_[ARG0],$_[ARG1], $_[ARG2] );
 		if ( $_[ARG0] =~ m!/(QUOTE)\s+(PASS\s+\d+)!i ) {
 			warn ">> $1 | $2\n";
 			$_[KERNEL]->post( $irc => lc($1) => $2);
@@ -1064,11 +1142,26 @@
 			"";
       0;                        # false for signals
     },
+	rss_response => sub {
+		my ($request_packet, $response_packet) = @_[ARG0, ARG1];
+		my $request_object  = $request_packet->[0];
+		my $response_object = $response_packet->[0];
+
+		my $row = delete( $_stat->{rss}->{fetch}->{ $request_object->uri } );
+		if ( $row ) {
+			$row->{xml} = $response_object->content;
+			rss_parse_xml( $row );
+		} else {
+			warn "## can't find rss->fetch for ", $request_object->uri;
+		}
+	},
    },
   );
 
 # http server
 
+_log "WEB archive at $url";
+
 my $httpd = POE::Component::Server::HTTP->new(
 	Port => $http_port,
 	PreHandler => {
@@ -1115,7 +1208,7 @@
 	$style .= ".col-${max_color} { background: $c }\n";
 	$max_color++;
 }
-warn "defined $max_color colors for users...\n";
+_log "WEB defined $max_color colors for users...";
 
 sub root_handler {
 	my ($request, $response) = @_;
@@ -1237,7 +1330,7 @@
 			$feed->add_entry( $feed_entry );
 
 		} else {
-			_log "unknown rss request $r_url";
+			_log "WEB unknown rss request $r_url";
 			$feed->title( "unknown $r_url" );
 			foreach my $c ( @commands ) {
 				my $feed_entry = XML::Feed::Entry->new($type);
@@ -1340,7 +1433,7 @@
 	<p>See <a href="/history">history</a> of all messages.</p>
 	</body></html>};
 
-	$response->content( $html );
+	$response->content( decode('utf-8',$html) );
 	warn "<< ", $request->method, " ", $request->uri, " created ", length($html), " bytes\n";
 	return RC_OK;
 }