/[SWISH-PlusPlus]/trunk/PlusPlus.pm

This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!

Diff of /trunk/PlusPlus.pm

Parent Directory | Revision Log | View Patch Patch

-revision 1 by dpavlin,
Fri Dec  3 13:31:43 2004 UTC
+revision 16 by dpavlin,
Sun Dec  5 21:06:48 2004 UTC
 Line 4 
 use 5.008004;
  use strict;
  use warnings;
- our $VERSION = '0.01';
+ our $VERSION = '0.10';
  use Carp;
+ use File::Temp qw/ tempdir /;
+ use BerkeleyDB;
+ #use YAML;
  =head1 NAME
-Line 20 
 SWISH::PlusPlus - Perl extension SWISH++
+Line 23 
 SWISH::PlusPlus - Perl extension SWISH++
  =head1 DESCRIPTION
  This is perl module to use SWISH++ indexer by Paul J. Lucas. SWISH++ is
- rewrite of swish-e in C++ with blazingly fast performance, but without
+ rewrite of swish-e in C++ which is extremly fast (thank to mmap), but without
- support for properties (which this module tries to fix)
+ support for properties (which this module tries to fix).
+ Implementation of this module is crafted after L<Plucene::Simple> and it
+ should be easy to replace Plucene with this module for increased
+ performance. However, this module is not plug-in replacement.
  =head1 METHODS
- =head2 open
+ =head2 new
  Create new indexing object.
-   my $i = new SWISH::PlusPlus(
+   my $i = SWISH::PlusPlus->new(
-         index => '/path/to/index',
+         index_dir => '/path/to/index',
+         index => 'index++',
+         search => 'search++',
+         debug => 1,
+         meta_in_body => 1,
+         use_stopwords => 1,
    );
- Options to open are following:
+ Options to new are following:
  =over 5
+ =item C<index_dir>
+ Path to directory in which index will be created.
  =item C<index>
- path to directory in which index will be created.
+ Full or partial path to SWISH++ index executable. By default, it's B<index>
+ for self-compiled version. If you use Debian GNU/Linux package specify
+ B<index++>. See C<Debian>.
+ =item C<search>
+ Full or partial path to SWISH++ search executable. By default, it's B<search>.
+ =item C<debug>
+ This option (off by default) will produce a lot of debugging output on
+ C<STDERR> prefixed by C<##>.
+ =item C<meta_in_body>
+ This option (off by default) enables to search content of meta fields
+ without specifing them (like they are in body of document). This will
+ somewhat increate index size.
+ =item C<use_stopwords>
+ Use built-in SWISH++ stop words. By default, they are disabled.
  =back
-Line 50 
 sub new {
+Line 87 
 sub new {
          my $self = {@_};
          bless($self, $class);
-         foreach (qw(index)) {
+         foreach (qw(index_dir)) {
                  croak "need $_" unless $self->{$_};
          }
-         if (! -e $self->{'index'}) {
+         my $index_dir = $self->{'index_dir'};
-                 mkdir $self->{'index'} || confess "can't create index ",$self->{'index'},": $!";
+         my $cwd;
+         chomp($cwd = `pwd`);
+         $self->{'cwd'} = $cwd || carp "can't get cwd!";
+         if ($index_dir !~ m#^/#) {
+                 $index_dir = "$cwd/$index_dir";
+                 print STDERR "## full path to index_dir: $index_dir\n" if ($self->{'debug'});
+                 $self->{'index_dir'} = $index_dir;
          }
+         if (! -e $index_dir) {
+                 mkdir $index_dir || confess "can't create index ",$self->{'index'},": $!";
+         }
+         # default executables
+         $self->{'index'} ||= 'index';
+         $self->{'search'} ||= 'search';
+         print STDERR "## new index_dir: ",$index_dir," index: ",$self->{'index'}, " search: ",$self->{'search'},"\n" if ($self->{'debug'});
          $self ? return $self : return undef;
  }
+ =head2 check_bin
+ Check if swish++ binaries specified in L<new> are available and verify
+ version signature.
+   if ($i->check_bin) {
+         print "swish++ binaries found\n";
+   };
+ It will also setup property
+   $i->{'version'}
+ which you can examine to see version.
+ =cut
+ sub check_bin {
+         my $self = shift;
+         my $i = `$self->{'index'} -V 2>&1` || confess "can't find '",$self->{'index'},"' binary";
+         my $s = `$self->{'search'} -V 2>&1` || confess "can't find '",$self->{'search'},"' binary";
+         chomp $i;
+         chomp $s;
+         confess $self->{'index'}," binary is not SWISH++" unless ($i =~ m/^SWISH\+\+/);
+         confess $self->{'search'}," binary is not SWISH++" unless ($s =~ m/^SWISH\+\+/);
+         if ($i eq $s) {
+                 $i =~ s/^SWISH\+\+\s+// || confess "can't strip SWISH++ from version";
+                 $self->{'version'} = $i;
+                 return 1;
+         } else  {
+                 carp "version difference: index is $i while search is $s";
+                 return;
+         }
+ }
+ =head2 index_document
+ Quick way to add simple data to index.
+   $i->index_document($key, $data);
+   $i->index_document( 42 => 'meaning of life' );
+ =cut
+ sub index_document {
+         my $self = shift;
+         my %doc = @_;
+         foreach my $id (keys %doc) {
+                 $self->_create_doc(
+                         path => $id,
+                         body => $doc{$id},
+                 );
+         }
+         return 1;
+ }
+ =head2 add
+ Add document with metadata to index.
+   $i->add(
+         path => 'path/to/document',
+         title => 'this is result title',
+         meta => {
+                 description => 'this is description meta tag',
+                 date => '2004-11-04',
+                 author => 'Dobrica Pavlinusic',
+         }
+         body => 'this is text without meta data',
+   );
+ This is thin wrapper round L<_create_doc>.
+ =cut
+ sub add {
+         my $self = shift;
+         $self->_create_doc(@_);
+         return 1;
+ }
+ =head2 search
+ Search your index.
+   my @results = $i->search("swhish query");
+ Returns array with result IDs.
+ =cut
+ sub search {
+         my $self = shift;
+         my $query = shift || return;
+         $self->finish_update;
+         $self->_tie_meta_db(DB_RDONLY);
+         my @results;
+         # escape double quotes in query for shell
+         $query =~ s/"/\\"/g;
+         my $open_cmd = $self->{'search'} .
+                 ' -i ' . $self->{'index_dir'}.'/index' .
+                 ' "' . $query . '"'.
+                 ' |';
+         print STDERR "## search: $open_cmd\n" if ($self->{'debug'});
+         open(SEARCH, $open_cmd) || confess "can't start $open_cmd: $!";
+         my $l;
+         while($l = <SEARCH>) {
+                 next if ($l =~ /^#/);
+                 chomp($l);
+                 print STDERR "## $l\n" if ($self->{'debug'});
+                 my ($rank,$path,$size,$title) = split(/ /,$l,4);
+                 $path =~ s#^\./##; # strip from path
+                 push @results, {
+                         rank => $rank,
+                         path => $path,
+                         size => $size,
+                         title => $title,
+                 }
+         }
+         close(SEARCH) || confess "can't close search";
+         #print STDERR "## results: ",Dump(@results),"\n" if ($self->{'debug'});
+         return @results;
+ }
+ =head2 property
+ Return stored meta property from result or result path.
+   print $i->property('path', 'title');
+   print $i->property($res->{'path'}, 'title');
+ =cut
+ sub property {
+         my $self = shift;
+         my ($path,$meta) = @_;
+         if ($path =~ m/^HASH/) {
+                 $path = $path->{'path'} || confess "can't find path in input data";
+         }
+         my $val = $self->{'meta_db'}->{"$path-$meta"};
+         print STDERR "## property $path-$meta: ",($val || 'undef'),"\n" if ($self->{'debug'});
+         return $val;
+ }
+ =head2 finish_update
+ This method will close index.
+   $i->finish_update;
+ It will be called on DESTROY when $i goes out of scope.
+ =cut
+ sub finish_update {
+         my $self = shift;
+         print STDERR "## finish_update\n" if ($self->{'debug'});
+         $self->_close_index && $self->_untie_meta_db;
+ }
+ sub DESTROY {
+         my $self = shift;
+         $self->finish_update;
+ }
+ =head1 PRIVATE METHODS
+ Private methods implement internals for creating temporary file needed for
+ swish++. You should have no need to call them directly, and they are here
+ just to have documentation.
+ =head2 _init_indexer
+ Create temporary directory in which files for indexing will be created and
+ start index process.
+   my $i->_init_indexer || die "can't start indexer";
+ It will also create empty file C<_stopwords_> to disable stop words.
+ =cut
+ sub _init_indexer {
+         my $self = shift;
+         return if ($self->{'_index_fh'});
+         my $tmp_dir = tempdir( CLEANUP => 1 ) || confess "can't create temporary directory: $!";
+         $self->{'tmp_dir'} = $tmp_dir;
+         chdir $tmp_dir || confess "can't chdir to ".$tmp_dir.": $!";
+         print STDERR "## tmp_dir: $tmp_dir" if ($self->{'debug'});
+         my $opt = "-v " . ($self->{'debug'} || '0');
+         unless ($self->{'use_stopwrods'}) {
+                 open(STOP, '>', "_stopwords_") || carp "can't create empty stopword file, skipping\n";
+                 print STOP "  ";
+                 close(STOP);
+                 $opt .= " -s _stopwords_";
+         }
+         my $index_dir = $self->{'index_dir'} || confess "no index_dir?";
+         my $open_cmd = '| '.$self->{'index'}.' '.$opt.' -e "html:*" -i '.$index_dir.'/index -';
+         print STDERR "## init_indexer: $open_cmd\n" if ($self->{'debug'});
+         open($self->{'_index_fh'}, $open_cmd) || confess "can't start index with $open_cmd: $!";
+         chdir $self->{'cwd'} || confess "can't chdir to ".$self->{'cwd'}.": $!";
+         $self->_tie_meta_db(DB_CREATE);
+         return $self->{'_index_fh'};
+ }
+ =head2 _tie_meta_db
+ Open BerkeleyDB database with meta properties.
+   $i->_tie_meta_db(DB_CREATE);
+   $i->_tie_meta_db(DB_RDONLY);
+ }
+ =cut
+ sub _tie_meta_db  {
+         my $self = shift;
+         my $flags = shift || confess "need DB_CREATE or DB_RDONLY";
+         return if ($self->{'_meta_db_flags'} && $self->{'_meta_db_flags'} == $flags);
+         print STDERR "## _tie_meta_db($flags)\n" if ($self->{'debug'});
+         $self->_untie_meta_db;
+         $self->{'_meta_db_flags'} = $flags;
+         my $file = $self->{'index_dir'}.'/meta.db';
+         tie %{$self->{'meta_db'}}, "BerkeleyDB::Hash",
+                 -Filename => $file,
+                 -Flags    => $flags
+         or confess "cannot open $file: $! $BerkeleyDB::Error\n" ;
+         return 1;
+ }
+ =head2 _untie_meta_db
+ Close BerkeleyDB database with meta properties.
+   $i->_untie_meta_db
+ =cut
+ sub _untie_meta_db {
+         my $self = shift;
+         return unless ($self->{'meta_db'});
+         print STDERR "## _untie_meta_db\n" if ($self->{'debug'});
+         untie %{$self->{'meta_db'}} || confess "can't untie!";
+         undef $self->{'meta_db'};
+         undef $self->{'_meta_db_flags'};
+         return 1;
+ }
+ =head2 _create_doc
+ Create temporary file and pass it's name to swish++
+   $i->_create_doc(
+         path => 'path/to/store/in/index',
+         title => 'this is title in results',
+         body => 'data to story in body tag',
+         meta => {
+                 'meta name' => 'data for this meta',
+                 'another' => 'again more data',
+         }
+   );
+ To delete document, just omit body and meta data.
+ =cut
+ sub _create_doc {
+         my $self = shift;
+         my $arg = {@_};
+         # open indexer if needed
+         $self->_init_indexer;
+         my $path = $self->{'tmp_dir'} || confess "no tmp_dir?";
+         my $id = $arg->{'path'} || confess "no path?";
+         $path .= "/$id";
+         print STDERR "## _create_doc: $path\n" if ($self->{'debug'});
+         open(TMP, '>', $path) || die "can't create temp file $path: $!";
+         print TMP '<html><head>';
+         $arg->{'body'} ||= '';
+         if ($arg->{'meta'}) {
+                 foreach my $name (keys %{$arg->{'meta'}}) {
+                         my $content = $arg->{'meta'}->{$name};
+                         print TMP qq{<meta name="$name" content="$content">};
+                         $arg->{'body'} .= " $content" if ($self->{'meta_in_body'});
+                         $self->{'meta_db'}->{"$id-$name"} = $content;
+                 }
+         }
+         my $title = $arg->{'title'};
+         if (defined($title)) {
+                 print TMP "<title>$title</title>";
+                 $arg->{'body'} .= " $title" if ($self->{'meta_in_body'});
+                 $self->{'meta_db'}->{"$id-title"} = $title;
+         }
+         print TMP '</head><body>' . $arg->{'body'} . '</body></html>';
+         close(TMP) || confess "can't close tmp file ".$arg->{'path'}.": $!";
+         print { $self->{'_index_fh'} } "$id\n";
+ }
+ =head2 _close_index
+ Close index after indexing.
+   $i->_close_index;
+ You have to close index before searching.
+ =cut
+ sub _close_index {
+         my $self = shift;
+         return unless ($self->{'_index_fh'});
+         print STDERR "## close index\n" if ($self->{'debug'});
+         close($self->{'_index_fh'}) || confess "can't close index: $!";
+         undef $self->{'_index_fh'};
+         return 1;
+ }
 ;
  __END__
-Line 69 
 __END__
+Line 504 
 __END__
  None by default.
- =head1 SEE ALSO
+ =head1 RELATED
- Mention other useful documentation such as the documentation of
+ =head2 Debian
- related modules or operating system documentation (such as man pages
- in UNIX), or any relevant external documentation such as RFCs or
- standards.
- If you have a mailing list set up for your module, mention it here.
+ Debian version of swish++ is often old (version 5 at moment of this writing
+ while version 6 is available in source code), so this module by default
+ uses executable names B<index> and B<search> for self-compiled version
+ instead of one from Debian package. See L<new> how to specify Debian
+ default binaries B<index++> and B<search++>.
+ =head2 SWISH++
+ Aside from very good rewrite in C++, SWISH++ is fatster because it has
+ claver heuristics about which data in input files are words to index and
+ which are not. It's based on English language and might be best choice if
+ you plan to install large amount of long text documents.
+ However, if you plan to index all data from structured storage (e.g. RDBMS)
+ you might want B<all> words from data to end up in index as opposed to just
+ those which look like English words. This is especially important if you
+ don't plan to index English texts with this module.
+ With distribution build versions of SWISH++ you might have problems with
+ disepearing words. To overcome this problem, you will have to compile and
+ configure SWISH++ yourself (because language characteristics are
+ compilation-time option).
+ Compilation of SWISH++ is easy process well described on project's web
+ pages. To see my very relaxed sample configuration take a look at C<swish++>
+ directory included in distribution.
+ =head2 SWISH++ config
+ C<config.h> located in C<swish++> directory of this distribution is relaxed
+ SWISH++ configuration that will index all words passed to it. This
+ configuration is needed for B<date test> because default configuration
+ doesn't recognize 2004-12-05 as date. Have in mind that your index size
+ might explode.
+ =head1 SEE ALSO
- If you have a web site set up for your module, mention it here.
+ C<swish++> web site L<http://homepage.mac.com/pauljlucas/software/swish/>
  =head1 AUTHOR
- Dobrica Pavlinusic, E<lt>dpavlin@E<gt>
+ Dobrica Pavlinusic, E<lt>dpavlin@rot13.orgE<gt>
  =head1 COPYRIGHT AND LICENSE

 Legend:



Removed from v.1
 


changed lines


 
Added in v.16
 Legend:



Removed from v.1
 


changed lines


 
Added in v.16
-Removed from v.1
+Added in v.16

	ViewVC Help
Powered by ViewVC 1.1.26