Grep/Search/KinoSearch.pm

package Grep::Search;

use strict;
use warnings;

use Data::Dump qw/dump/;
use Lucene;
use Jifty::Util;

my $index_path = Jifty::Util->app_root . '/var/lucene';

my ( $analyzer, $store, $writer );

my $debug = 1;

sub create {

        my $create = 0;
        if (! -e "$index_path/segments") {
                $create = 1;
                Jifty->log->debug("create index $index_path") unless ($store);
        } else {
                Jifty->log->debug("open index: $index_path") unless ($store);
        }
        return $create;
}

sub analyzer {
        my $self = shift;
        $analyzer ||= new Lucene::Analysis::Standard::StandardAnalyzer();
        return $analyzer;
}

sub store {
        my $self = shift;

        $store ||= Lucene::Store::FSDirectory->getDirectory( $index_path, $self->create );
        return $store;
}

sub writer {
        my $self = shift;
        $writer ||= new Lucene::Index::IndexWriter( $self->store, $self->analyzer, $self->create );
        return $writer;
}

=head2 add

  Grep::Search->add( $record );

=cut

sub add {
        my $self = shift;

        my $i = shift or die "no record to add";

        die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));

        my $pk = { $i->primary_keys };

        my $doc = new Lucene::Document;

        my @columns = map { $_->name } $i->columns;

        foreach my $c ( @columns ) {

                my $v = $i->$c;

                if ( ref($v) ne '' ) {

                        foreach my $f_c ( qw/id name title/ ) {
                                if ( $i->$c->can( $f_c ) ) {
                                        my $f_v = $i->$c->$f_c || $i->$c->{values}->{ $f_c };
                                        my $col = $c . '_' . $f_c;
                                        if ( $f_v ) {
                                                warn "  # $col = $f_v\n" if ($debug);
                                                $doc->add(Lucene::Document::Field->Text( $col, $f_v ));
                                        } else {
                                                warn "  . $col is NULL\n" if ($debug);
                                        }
                                }
                        }

                        if ($v->isa('Jifty::DateTime')) {
                                warn "  d $c = $v\n" if ($debug);
                                $doc->add(Lucene::Document::Field->Keyword( $c, "$v" ));
                        } else {
                                warn "  s $c = $v [",ref($v),"]\n" if ($debug);
                        }
                        next;
                }

                next if (! defined($v) || $v eq '');

                $v =~ s/<[^>]+>/ /gs;

                if ( defined( $pk->{$c} ) ) {
                        $doc->add(Lucene::Document::Field->Keyword( $c, $v ));
                        warn "  * $c = $v\n" if ($debug);
                } else {
                        $doc->add(Lucene::Document::Field->Text( $c, $v ));
                        warn "  + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
                }
        }

        $self->writer->addDocument($doc);

        Jifty->log->debug("added ", $i->id, " to index");
}

=head2

  my $ItemCollection = Grep::Search->collection( 'search query' );

=cut

sub collection {
        my $self = shift;

        my $q = shift or die "no q?";

        my $searcher = new Lucene::Search::IndexSearcher($self->store);
        my $parser = new Lucene::QueryParser("content", $self->analyzer);
        my $query = $parser->parse( $q );

        Jifty->log->debug("searching for '$q'");

        my $hits = $searcher->search($query);
        my $num_hits = $hits->length();

        Jifty->log->debug("found $num_hits results");

        my $collection = Grep::Model::ItemCollection->new();

        my @results;

        for ( my $i = 0; $i < $num_hits; $i++ ) {

                my $doc = $hits->doc( $i );

                my $score = $hits->score($i);
                my $title = $doc->get("title");
                my $id = $doc->get("id");

                warn "## $i $score $title\n";

                my $item = Grep::Model::Item->new();
                my ($ok,$msg) = $item->load_by_cols( id => $id );

                if ( $ok ) {
                        $collection->add_record( $item );
                } else {
                        warn "can't load item $id\n";
                }

        }

        undef $hits;
        undef $query;
        undef $parser;
        undef $searcher;

        return $collection;
}

=head2 finish

  Grep::Search->finish

=cut

sub finish {
        my $self = shift;
        if ($writer) {
                warn "closing index\n";
                $writer->close;
        }
        undef $writer;
}

sub _signal {
        my $s = shift;
        warn "catched SIG $s\n";
        finish();
        exit(0);
}

$SIG{'__DIE__'} = \&_signal;
$SIG{'INT'} = \&_signal;
$SIG{'QUIT'} = \&_signal;


=head2 snippet

  my $short = $self->snippet( 50, $text );


=cut

sub snippet {
        my $self = shift;

        my $len = shift or die "no len?";
        my $m = join(" ", @_);

        $m =~ s/\s+/ /gs;

        if (length($m) > $len) {
                return substr($m,0,$len) . '...';
        } else {
                return $m;
        }
}

1;
1	dpavlin	47	package Grep::Search;
2
3			use strict;
4			use warnings;
5
6			use Data::Dump qw/dump/;
7			use Lucene;
8			use Jifty::Util;
9
10			my $index_path = Jifty::Util->app_root . '/var/lucene';
11
12			my ( $analyzer, $store, $writer );
13
14	dpavlin	53	my $debug = 1;
15	dpavlin	47
16	dpavlin	49	sub create {
17	dpavlin	47
18			my $create = 0;
19			if (! -e "$index_path/segments") {
20			$create = 1;
21	dpavlin	49	Jifty->log->debug("create index $index_path") unless ($store);
22	dpavlin	47	} else {
23	dpavlin	49	Jifty->log->debug("open index: $index_path") unless ($store);
24	dpavlin	47	}
25	dpavlin	49	return $create;
26	dpavlin	47	}
27
28	dpavlin	49	sub analyzer {
29			my $self = shift;
30			$analyzer \|\|= new Lucene::Analysis::Standard::StandardAnalyzer();
31			return $analyzer;
32			}
33
34			sub store {
35			my $self = shift;
36
37			$store \|\|= Lucene::Store::FSDirectory->getDirectory( $index_path, $self->create );
38			return $store;
39			}
40
41			sub writer {
42			my $self = shift;
43			$writer \|\|= new Lucene::Index::IndexWriter( $self->store, $self->analyzer, $self->create );
44			return $writer;
45			}
46
47	dpavlin	47	=head2 add
48
49			Grep::Search->add( $record );
50
51			=cut
52
53			sub add {
54			my $self = shift;
55
56			my $i = shift or die "no record to add";
57
58			die "record not Jifty::Record but ", ref $i unless ($i->isa('Jifty::Record'));
59
60			my $pk = { $i->primary_keys };
61
62			my $doc = new Lucene::Document;
63
64			my @columns = map { $_->name } $i->columns;
65
66			foreach my $c ( @columns ) {
67
68			my $v = $i->$c;
69
70			if ( ref($v) ne '' ) {
71	dpavlin	53
72			foreach my $f_c ( qw/id name title/ ) {
73			if ( $i->$c->can( $f_c ) ) {
74			my $f_v = $i->$c->$f_c \|\| $i->$c->{values}->{ $f_c };
75			my $col = $c . '_' . $f_c;
76			if ( $f_v ) {
77			warn " # $col = $f_v\n" if ($debug);
78			$doc->add(Lucene::Document::Field->Text( $col, $f_v ));
79			} else {
80			warn " . $col is NULL\n" if ($debug);
81			}
82			}
83			}
84
85			if ($v->isa('Jifty::DateTime')) {
86	dpavlin	47	warn " d $c = $v\n" if ($debug);
87			$doc->add(Lucene::Document::Field->Keyword( $c, "$v" ));
88			} else {
89			warn " s $c = $v [",ref($v),"]\n" if ($debug);
90			}
91			next;
92			}
93
94			next if (! defined($v) \|\| $v eq '');
95
96			$v =~ s/<[^>]+>/ /gs;
97
98			if ( defined( $pk->{$c} ) ) {
99			$doc->add(Lucene::Document::Field->Keyword( $c, $v ));
100			warn " * $c = $v\n" if ($debug);
101			} else {
102			$doc->add(Lucene::Document::Field->Text( $c, $v ));
103	dpavlin	53	warn " + $c = ", $self->snippet( 50, $v ), "\n" if ($debug);
104	dpavlin	47	}
105			}
106
107	dpavlin	49	$self->writer->addDocument($doc);
108	dpavlin	47
109			Jifty->log->debug("added ", $i->id, " to index");
110			}
111
112			=head2
113
114			my $ItemCollection = Grep::Search->collection( 'search query' );
115
116			=cut
117
118			sub collection {
119			my $self = shift;
120
121			my $q = shift or die "no q?";
122
123	dpavlin	49	my $searcher = new Lucene::Search::IndexSearcher($self->store);
124			my $parser = new Lucene::QueryParser("content", $self->analyzer);
125	dpavlin	47	my $query = $parser->parse( $q );
126
127			Jifty->log->debug("searching for '$q'");
128
129			my $hits = $searcher->search($query);
130			my $num_hits = $hits->length();
131
132			Jifty->log->debug("found $num_hits results");
133
134			my $collection = Grep::Model::ItemCollection->new();
135
136			my @results;
137
138			for ( my $i = 0; $i < $num_hits; $i++ ) {
139
140			my $doc = $hits->doc( $i );
141
142			my $score = $hits->score($i);
143			my $title = $doc->get("title");
144			my $id = $doc->get("id");
145
146			warn "## $i $score $title\n";
147
148			my $item = Grep::Model::Item->new();
149			my ($ok,$msg) = $item->load_by_cols( id => $id );
150
151			if ( $ok ) {
152			$collection->add_record( $item );
153			} else {
154			warn "can't load item $id\n";
155			}
156
157			}
158
159			undef $hits;
160			undef $query;
161			undef $parser;
162			undef $searcher;
163
164			return $collection;
165			}
166
167			=head2 finish
168
169			Grep::Search->finish
170
171			=cut
172
173			sub finish {
174			my $self = shift;
175			if ($writer) {
176			warn "closing index\n";
177			$writer->close;
178			}
179			undef $writer;
180			}
181
182			sub _signal {
183			my $s = shift;
184			warn "catched SIG $s\n";
185			finish();
186			exit(0);
187			}
188
189			$SIG{'__DIE__'} = \&_signal;
190			$SIG{'INT'} = \&_signal;
191			$SIG{'QUIT'} = \&_signal;
192
193	dpavlin	53
194			=head2 snippet
195
196			my $short = $self->snippet( 50, $text );
197
198
199			=cut
200
201			sub snippet {
202			my $self = shift;
203
204			my $len = shift or die "no len?";
205			my $m = join(" ", @_);
206
207			$m =~ s/\s+/ /gs;
208
209			if (length($m) > $len) {
210			return substr($m,0,$len) . '...';
211			} else {
212			return $m;
213			}
214			}
215
216	dpavlin	47	1;