--- trunk/MWS.pm	2004/05/06 12:40:11	12
+++ trunk/MWS.pm	2004/05/08 00:54:16	21
@@ -8,6 +8,14 @@
 
 use Mail::Box::Manager;
 use Config::IniFiles;
+use POSIX qw(strftime);
+use Text::Autoformat;
+use Text::Iconv;
+use Text::Unaccent;
+use Date::Parse;
+use POSIX qw(strftime);
+use MIME::Base64;
+
 #use MWS_plucene;
 use MWS_swish;
 
@@ -21,10 +29,9 @@
 
 our $VERSION = '1.00';
 
-
 my $folder;	# placeholder for folders
 
-my $debug = 1;
+my $debug = 2;
 
 sub new {
 	my $class = shift;
@@ -43,24 +50,135 @@
 	# placeholder for opened folders
 	$self->{folder} = {};
 
+	$self->{wrap_margin} = $self->{config}->val('global', 'wrap_margin');
+	$self->{max_results} = $self->{config}->val('global', 'max_results') || 100;
+	$self->reset_counters;
+
 	return $self;
 }
 
+sub normalize_string {
+	my $self = shift;
+
+	my $v = shift || return;
+
+	$v = unac_string('ISO-8859-2', $v);
+	$v = join('',sort split(/\s+/,$v));
+	$v =~ s/\W+//g;
+
+	return $v;
+}
+
+# reset tables for search results
+sub reset_counters {
+	my $self = shift;
+
+	$self->{counter} = {};
+
+#	foreach my $c (qw(thread from to cc bcc lists links att)) {
+#		$self->{counter}->{$c} = {};
+#	}
+
+}
+
+sub add_counter($$) {
+	my $self = shift;
+
+	my ($c,$v) = @_;
+	my $k = $self->normalize_string($v);
+
+	$self->{counter}->{$c}->{$k}->{name} = $v;
+	return $self->{counter}->{$c}->{$k}->{usage}++;
+}
+
+sub yyyymmdd {
+	my $self = shift;
+
+	my $t = shift || time;
+
+	my (undef,undef,undef,$dd,$mm,$yyyy) = localtime($t);
+	$mm++;
+	$yyyy+=1900;
+	return ($yyyy,$mm,$dd);
+}
+
+sub fmtdate {
+	my $self = shift;
+
+	my @out;
+	my @formats = qw(%04d %02d %02d);
+	while (my $v = shift) {
+		my $f = shift @formats;
+		push @out, sprintf($f, $v);
+	}
+
+print STDERR "fmtdate: ",join('|',@out),"\n";
+
+	return (wantarray ? @out : join("-",@out));
+}
+
+sub add_counter_calendar($) {
+	my $self = shift;
+
+	my $t = shift || croak "add_counter_calendar without argument!";
+	
+	my ($yyyy,$mm,$dd) = $self->fmtdate($self->yyyymmdd($t));
+
+	return $self->{counter}->{calendar}->{"$yyyy-$mm"}->{$dd}++;
+}
+
+
+sub counter {
+	my $self = shift;
+
+	my $c = shift || return;
+
+	return if (! $self->{counter}->{$c});
+
+	return $self->{counter}->{$c};
+}
+
+sub mbox_name2path {
+	my $self = shift;
+
+	my $mbox = shift || croak "folder_name2path needs mbox name";
+
+	return $self->{config}->val('folders', $mbox) || croak "comeone removed folder $mbox from config?";
+}
+
 sub open_folder {
 	my $self = shift;
 
 	my $mbox = shift || croak "open_folder needs mbox name";
 
 	if (! $self->{folder}->{$mbox}) {
-		my $mbox_path = $self->{config}->val('folders', $mbox) || croak "comeone removed folder $mbox from config?";
+		my $mbox_path = $self->mbox_name2path($mbox);
+
+		print STDERR "about to open_folder($mbox)\n" if ($debug == 2);
+
 		$self->{folder}->{$mbox} = $self->{mgr}->open($mbox_path) || croak "can't open folder $mbox at '$mbox_path': $!";
-		print STDERR "## open($mbox)\n" if ($debug);
+
+		print STDERR "open_folder($mbox) ok\n" if ($debug);
 	}
 
+	$self->{fetch_count} = 0;
+
 	return $self->{folder}->{$mbox};
 
 }
 
+sub close_folder {
+	my $self = shift;
+
+	my $mbox = shift || croak "open_folder needs mbox name";
+
+	$self->{folder}->{$mbox}->close(write => 'NEVER') || croak "can't close folder $mbox";
+
+	# XXX this is rather agressive!!!
+	$self->{folder} = {};
+	return 
+}
+
 sub fetch_message {
 	my $self = shift;
 
@@ -68,39 +186,96 @@
 	my ($mbox,$id) = split(/ /,$mbox_id);
 
 	# return message with ID
-	return $self->open_folder($mbox)->find($id) ||
+	print STDERR "fetch $id from $mbox\n" if ($debug);
+
+	if ($self->{fetch_count}++ > 100) {
+		$self->close_folder($mbox);
+		print STDERR "close_folder($mbox) forced on ",$self->{fetch_count},"iteration\n";
+	}
+	
+	my $msg = $self->open_folder($mbox)->find($id);
+	if ($msg) {
+		return $msg;
+	} else {
 		print STDERR "can't find message $id in $mbox. Time to re-index?\n";
+		return;
+	}
 }
 
 
 sub search {
 	my $self = shift;
 
-	my $s = shift || carp "search called without argument!";
+	carp "search called without argument!" if (! @_);
+
+	$self->reset_counters;
 
-	my @index_ids = $self->search_index($s);
+	print STDERR "search(",join(" ",@_),")\n" if ($debug == 2);
+	my @index_ids = $self->search_index(@_);
 
 	$self->{'index_ids'} = \@index_ids;
 
-	my $results = $#index_ids + 1;
-	$self->{'results'} = $results;
+	#my $results = $#index_ids + 1;
+	#$self->{'results'} = $results;
+	
+	my $results = $self->{'total_hits'} || ($#index_ids + 1);
 
 	$self->{'curr_result'} = 0;
 
+	print STDERR "$results results\n" if ($debug == 2);
+
 	return $results || 'error';
 }
 
+sub decode_qp($) {
+	my $self = shift;
+
+	my $tmp = shift || return;
+
+	sub decode($$$) {
+		my ($cp,$enc,$qp) = @_;
+
+		print STDERR "decode($cp,$qp) -> " if ($debug == 2);
+
+		if (uc($enc) eq "Q") {
+			$qp =~ s/=([a-f0-9][a-f0-9])/chr(hex($1))/ieg;
+			$qp =~ s/_/ /g;
+		} elsif (uc($enc) eq "B") {
+			$qp = decode_base64($qp);
+		} else {
+			croak "unsupported encoding '$enc' in decode_qp\n";
+			return $qp;
+		}
+
+		print STDERR "$qp -> " if ($debug == 2);
+
+		my $iconv = Text::Iconv->new($cp,'ISO-8859-2');
+		return $iconv->convert($qp) || '';
+	}
+
+	$tmp =~ s/=\?([^\?]+)\?([QB])\?(.+?)\?=/decode($1,$2,$3)/ige;
+	$tmp =~ s/^\s*["']+(.*?)["']+\s*$/$1/g;
+	#print STDERR "$tmp\n" if ($debug == 2);
+	return $tmp;
+}
+
 sub unroll($$$) {
+	my $self = shift;
+
 	my ($message,$part,$sub) = @_;
 
 	my @arr;
 
+	return if (! $message->$part);
+
 	foreach my $from ($message->$part) {
-		my $tmp = $from->$sub;
-		$tmp =~ s/^\s*["'](.*)["']\s*$/$1/;
+		my $tmp = $from->$sub || next;
+
+		$tmp = $self->decode_qp($tmp);
 		push @arr, $tmp;
 	}
-	return \@arr;
+
+	return @arr;
 }
 
 sub fetch_all_results {
@@ -108,12 +283,15 @@
 
 	croak "results called before search!" if (! $self->{'index_ids'});
 
+	print STDERR "fetch_all_results [",scalar @{$self->{'index_ids'}},"]\n" if ($debug == 2);
+
 	my @arr;
 
 	foreach my $id (@{$self->{'index_ids'}}) {
 		push @arr, $self->fetch_result_by_id($id);
 	}
 
+
 	return @arr;
 }
 
@@ -127,6 +305,8 @@
 	my $curr = $self->{'curr_result'}++;
 
 	my $id = $self->{'index_ids'}->[$curr];
+	
+	print STDERR "fetch_result: $curr = $id\n" if ($debug == 2);
 
 	return $self->fetch_result_by_id($id);
 }
@@ -135,15 +315,28 @@
 	my $self = shift;
 	my $message = shift || croak "plain_text_body needs message!";
 
+	my $body;
+
 	if (! $message->isMultipart) {
-		return $message->decoded->string;
+		$body = $message->decoded->string;
 	} else {
 		foreach my $part ($message->parts) {
 			if ($part->body->mimeType eq 'text/plain') {
-				return $part->decoded->string;
+				$body = $part->decoded->string;
+				last;
 			}
 		}
 	}
+
+	# reformat with Text::Autoformat
+	my $wrap = $self->{wrap_margin};
+	if ($wrap && $body && $body =~ m/^.{$wrap}..*$/m) {
+		$body =~ s/[\r\n]/\n/gs;
+		$body = autoformat($body, {right=>$wrap, all=>1});
+		$body .="\n[reformated using autoformat, margin at $wrap]" if ($debug == 2);
+	}
+
+	return $body;
 }
 
 
@@ -152,14 +345,37 @@
 
 	my $id = shift || return;
 
-	my $message = $self->fetch_message($id);
+	my $row = $self->{cache}->{$id};
+
+	if (! $row) {
 
-	my $row;
+		print STDERR "fetch_result_by_id($id) not in cache, hitting disk\n" if ($debug == 2);
 
-	$row->{'id'} = $id;
-	$row->{'from'} = unroll($message,'from','phrase');
-	$row->{'subject'} = $message->subject;
-	$row->{'body'} = $self->plain_text_body($message);
+		my $message = $self->fetch_message($id) || return;
+
+		$row->{'id'} = $id;
+
+		foreach my $p (qw(from to cc bcc)) {
+			foreach my $v ($self->unroll($message,'from','phrase')) {
+				push @{$row->{$p}},$v;
+				$self->add_counter($p,$v);
+			}
+		}
+		$row->{'subject'} = $self->decode_qp($message->subject);
+		$row->{'body'} = $self->plain_text_body($message);
+		my $utime = str2time($message->date);
+
+		$row->{'date_utime'} = $utime;
+
+		$row->{'date'} = strftime("%Y-%m-%d %H:%M:%S", localtime($utime));
+		$self->add_counter_calendar($utime);
+
+		# XXX store in cache?
+		$self->{cache}->{$id} = $row;
+		print STDERR "$id stored in cache\n" if ($debug == 2);
+	} else {
+		print STDERR "fetch_result_by_id($id) in cache\n" if ($debug == 2);
+	}
 
 	return $row;