/[SWISH-PlusPlus]/trunk/PlusPlus.pm

This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!

Diff of /trunk/PlusPlus.pm

Parent Directory | Revision Log | View Patch Patch

-revision 5 by dpavlin,
Fri Dec  3 21:48:15 2004 UTC
+revision 16 by dpavlin,
Sun Dec  5 21:06:48 2004 UTC
 Line 4 
 use 5.008004;
  use strict;
  use warnings;
- our $VERSION = '0.02';
+ our $VERSION = '0.10';
  use Carp;
  use File::Temp qw/ tempdir /;
+ use BerkeleyDB;
+ #use YAML;
  =head1 NAME
-Line 30 
 performance. However, this module is not
+Line 32 
 performance. However, this module is not
  =head1 METHODS
- =head2 open
+ =head2 new
  Create new indexing object.
-   my $i = SWISH::PlusPlus->open(
+   my $i = SWISH::PlusPlus->new(
          index_dir => '/path/to/index',
          index => 'index++',
          search => 'search++',
+         debug => 1,
+         meta_in_body => 1,
+         use_stopwords => 1,
    );
- Options to open are following:
+ Options to new are following:
  =over 5
-Line 58 
 B<index++>. See C<Debian>.
+Line 63 
 B<index++>. See C<Debian>.
  Full or partial path to SWISH++ search executable. By default, it's B<search>.
+ =item C<debug>
+ This option (off by default) will produce a lot of debugging output on
+ C<STDERR> prefixed by C<##>.
+ =item C<meta_in_body>
+ This option (off by default) enables to search content of meta fields
+ without specifing them (like they are in body of document). This will
+ somewhat increate index size.
+ =item C<use_stopwords>
+ Use built-in SWISH++ stop words. By default, they are disabled.
  =back
  =cut
- sub open {
+ sub new {
          my $class = shift;
          my $self = {@_};
          bless($self, $class);
-Line 71 
 sub open {
+Line 91 
 sub open {
                  croak "need $_" unless $self->{$_};
          }
-         if (! -e $self->{'index_dir'}) {
+         my $index_dir = $self->{'index_dir'};
-                 mkdir $self->{'index_dir'} || confess "can't create index ",$self->{'index'},": $!";
+         my $cwd;
+         chomp($cwd = `pwd`);
+         $self->{'cwd'} = $cwd || carp "can't get cwd!";
+         if ($index_dir !~ m#^/#) {
+                 $index_dir = "$cwd/$index_dir";
+                 print STDERR "## full path to index_dir: $index_dir\n" if ($self->{'debug'});
+                 $self->{'index_dir'} = $index_dir;
+         }
+         if (! -e $index_dir) {
+                 mkdir $index_dir || confess "can't create index ",$self->{'index'},": $!";
          }
          # default executables
          $self->{'index'} ||= 'index';
          $self->{'search'} ||= 'search';
+         print STDERR "## new index_dir: ",$index_dir," index: ",$self->{'index'}, " search: ",$self->{'search'},"\n" if ($self->{'debug'});
          $self ? return $self : return undef;
  }
  =head2 check_bin
- Check if swish++ binaries specified in L<open> are available and verify
+ Check if swish++ binaries specified in L<new> are available and verify
  version signature.
    if ($i->check_bin) {
-Line 113 
 sub check_bin {
+Line 147 
 sub check_bin {
          confess $self->{'search'}," binary is not SWISH++" unless ($s =~ m/^SWISH\+\+/);
          if ($i eq $s) {
+                 $i =~ s/^SWISH\+\+\s+// || confess "can't strip SWISH++ from version";
                  $self->{'version'} = $i;
                  return 1;
          } else  {
-Line 146 
 sub index_document {
+Line 181 
 sub index_document {
          return 1;
  }
+ =head2 add
+ Add document with metadata to index.
+   $i->add(
+         path => 'path/to/document',
+         title => 'this is result title',
+         meta => {
+                 description => 'this is description meta tag',
+                 date => '2004-11-04',
+                 author => 'Dobrica Pavlinusic',
+         }
+         body => 'this is text without meta data',
+   );
+ This is thin wrapper round L<_create_doc>.
+ =cut
+ sub add {
+         my $self = shift;
+         $self->_create_doc(@_);
+         return 1;
+ }
+ =head2 search
+ Search your index.
+   my @results = $i->search("swhish query");
+ Returns array with result IDs.
+ =cut
+ sub search {
+         my $self = shift;
+         my $query = shift || return;
+         $self->finish_update;
+         $self->_tie_meta_db(DB_RDONLY);
+         my @results;
+         # escape double quotes in query for shell
+         $query =~ s/"/\\"/g;
+         my $open_cmd = $self->{'search'} .
+                 ' -i ' . $self->{'index_dir'}.'/index' .
+                 ' "' . $query . '"'.
+                 ' |';
+         print STDERR "## search: $open_cmd\n" if ($self->{'debug'});
+         open(SEARCH, $open_cmd) || confess "can't start $open_cmd: $!";
+         my $l;
+         while($l = <SEARCH>) {
+                 next if ($l =~ /^#/);
+                 chomp($l);
+                 print STDERR "## $l\n" if ($self->{'debug'});
+                 my ($rank,$path,$size,$title) = split(/ /,$l,4);
+                 $path =~ s#^\./##; # strip from path
+                 push @results, {
+                         rank => $rank,
+                         path => $path,
+                         size => $size,
+                         title => $title,
+                 }
+         }
+         close(SEARCH) || confess "can't close search";
+         #print STDERR "## results: ",Dump(@results),"\n" if ($self->{'debug'});
+         return @results;
+ }
+ =head2 property
+ Return stored meta property from result or result path.
+   print $i->property('path', 'title');
+   print $i->property($res->{'path'}, 'title');
+ =cut
+ sub property {
+         my $self = shift;
+         my ($path,$meta) = @_;
+         if ($path =~ m/^HASH/) {
+                 $path = $path->{'path'} || confess "can't find path in input data";
+         }
+         my $val = $self->{'meta_db'}->{"$path-$meta"};
+         print STDERR "## property $path-$meta: ",($val || 'undef'),"\n" if ($self->{'debug'});
+         return $val;
+ }
+ =head2 finish_update
+ This method will close index.
+   $i->finish_update;
+ It will be called on DESTROY when $i goes out of scope.
+ =cut
+ sub finish_update {
+         my $self = shift;
+         print STDERR "## finish_update\n" if ($self->{'debug'});
+         $self->_close_index && $self->_untie_meta_db;
+ }
+ sub DESTROY {
+         my $self = shift;
+         $self->finish_update;
+ }
  =head1 PRIVATE METHODS
  Private methods implement internals for creating temporary file needed for
  swish++. You should have no need to call them directly, and they are here
  just to have documentation.
- =head2 _init_index
+ =head2 _init_indexer
  Create temporary directory in which files for indexing will be created and
  start index process.
-   my $i->_init_index || die "can't start indexer";
+   my $i->_init_indexer || die "can't start indexer";
+ It will also create empty file C<_stopwords_> to disable stop words.
  =cut
- sub _init_index {
+ sub _init_indexer {
          my $self = shift;
-         $self->{'tmp_dir'} = tempdir( CLEANUP => 1 ) || confess "can't create temporary directory: $!";
+         return if ($self->{'_index_fh'});
+         my $tmp_dir = tempdir( CLEANUP => 1 ) || confess "can't create temporary directory: $!";
+         $self->{'tmp_dir'} = $tmp_dir;
-         my $opt = "-v 4";
+         chdir $tmp_dir || confess "can't chdir to ".$tmp_dir.": $!";
+         print STDERR "## tmp_dir: $tmp_dir" if ($self->{'debug'});
+         my $opt = "-v " . ($self->{'debug'} || '0');
+         unless ($self->{'use_stopwrods'}) {
+                 open(STOP, '>', "_stopwords_") || carp "can't create empty stopword file, skipping\n";
+                 print STOP "  ";
+                 close(STOP);
+                 $opt .= " -s _stopwords_";
+         }
+         my $index_dir = $self->{'index_dir'} || confess "no index_dir?";
+         my $open_cmd = '| '.$self->{'index'}.' '.$opt.' -e "html:*" -i '.$index_dir.'/index -';
+         print STDERR "## init_indexer: $open_cmd\n" if ($self->{'debug'});
+         open($self->{'_index_fh'}, $open_cmd) || confess "can't start index with $open_cmd: $!";
+         chdir $self->{'cwd'} || confess "can't chdir to ".$self->{'cwd'}.": $!";
+         $self->_tie_meta_db(DB_CREATE);
+         return $self->{'_index_fh'};
+ }
-         my $open_cmd = '| index '.$opt.' -e "html:*" -i '.$self->{'index_dir'}.'/index -';
+ =head2 _tie_meta_db
-         chdir $self->{'tmp_dir'} || confess "can't chdir to ".$self->{'tmp_dir'}.": $!";
+ Open BerkeleyDB database with meta properties.
-         CORE::open($self->{'index_fh'}, $open_cmd) || confess "can't start index with $open_cmd: $!";
+   $i->_tie_meta_db(DB_CREATE);
+   $i->_tie_meta_db(DB_RDONLY);
-         return $self->{'index_fh'};
+ }
+ =cut
+ sub _tie_meta_db  {
+         my $self = shift;
+         my $flags = shift || confess "need DB_CREATE or DB_RDONLY";
+         return if ($self->{'_meta_db_flags'} && $self->{'_meta_db_flags'} == $flags);
+         print STDERR "## _tie_meta_db($flags)\n" if ($self->{'debug'});
+         $self->_untie_meta_db;
+         $self->{'_meta_db_flags'} = $flags;
+         my $file = $self->{'index_dir'}.'/meta.db';
+         tie %{$self->{'meta_db'}}, "BerkeleyDB::Hash",
+                 -Filename => $file,
+                 -Flags    => $flags
+         or confess "cannot open $file: $! $BerkeleyDB::Error\n" ;
+         return 1;
+ }
+ =head2 _untie_meta_db
+ Close BerkeleyDB database with meta properties.
+   $i->_untie_meta_db
+ =cut
+ sub _untie_meta_db {
+         my $self = shift;
+         return unless ($self->{'meta_db'});
+         print STDERR "## _untie_meta_db\n" if ($self->{'debug'});
+         untie %{$self->{'meta_db'}} || confess "can't untie!";
+         undef $self->{'meta_db'};
+         undef $self->{'_meta_db_flags'};
+         return 1;
  }
  =head2 _create_doc
-Line 183 
 Create temporary file and pass it's name
+Line 419 
 Create temporary file and pass it's name
    $i->_create_doc(
          path => 'path/to/store/in/index',
+         title => 'this is title in results',
          body => 'data to story in body tag',
          meta => {
                  'meta name' => 'data for this meta',
-Line 190 
 Create temporary file and pass it's name
+Line 427 
 Create temporary file and pass it's name
          }
    );
+ To delete document, just omit body and meta data.
  =cut
  sub _create_doc {
-Line 198 
 sub _create_doc {
+Line 437 
 sub _create_doc {
          my $arg = {@_};
          # open indexer if needed
-         $self->{'index_fh'} ||= $self->_init_index;
+         $self->_init_indexer;
          my $path = $self->{'tmp_dir'} || confess "no tmp_dir?";
+         my $id = $arg->{'path'} || confess "no path?";
+         $path .= "/$id";
+         print STDERR "## _create_doc: $path\n" if ($self->{'debug'});
+         open(TMP, '>', $path) || die "can't create temp file $path: $!";
-         CORE::open(TMP, '>', $arg->{'path'}) || die "can't create temp file ".$arg->{'path'}.": $!";
+         print TMP '<html><head>';
-         print TMP '<html>';
+         $arg->{'body'} ||= '';
          if ($arg->{'meta'}) {
-                 confess "not yet implemented";
+                 foreach my $name (keys %{$arg->{'meta'}}) {
+                         my $content = $arg->{'meta'}->{$name};
+                         print TMP qq{<meta name="$name" content="$content">};
+                         $arg->{'body'} .= " $content" if ($self->{'meta_in_body'});
+                         $self->{'meta_db'}->{"$id-$name"} = $content;
+                 }
          }
-         print TMP '<body>' . ($arg->{'body'} || '') . '</body></html>';
+         my $title = $arg->{'title'};
+         if (defined($title)) {
+                 print TMP "<title>$title</title>";
+                 $arg->{'body'} .= " $title" if ($self->{'meta_in_body'});
+                 $self->{'meta_db'}->{"$id-title"} = $title;
+         }
+         print TMP '</head><body>' . $arg->{'body'} . '</body></html>';
          close(TMP) || confess "can't close tmp file ".$arg->{'path'}.": $!";
-         print { $self->{'index_fh'} } $arg->{'path'}."\n";
+         print { $self->{'_index_fh'} } "$id\n";
+ }
+ =head2 _close_index
+ Close index after indexing.
+   $i->_close_index;
+ You have to close index before searching.
+ =cut
+ sub _close_index {
+         my $self = shift;
+         return unless ($self->{'_index_fh'});
+         print STDERR "## close index\n" if ($self->{'debug'});
+         close($self->{'_index_fh'}) || confess "can't close index: $!";
+         undef $self->{'_index_fh'};
+         return 1;
  }
 ;
-Line 231 
 None by default.
+Line 511 
 None by default.
  Debian version of swish++ is often old (version 5 at moment of this writing
  while version 6 is available in source code), so this module by default
  uses executable names B<index> and B<search> for self-compiled version
- instead of one from Debian package. See L<open> how to specify Debian
+ instead of one from Debian package. See L<new> how to specify Debian
  default binaries B<index++> and B<search++>.
  =head2 SWISH++
-Line 255 
 Compilation of SWISH++ is easy process w
+Line 535 
 Compilation of SWISH++ is easy process w
  pages. To see my very relaxed sample configuration take a look at C<swish++>
  directory included in distribution.
+ =head2 SWISH++ config
+ C<config.h> located in C<swish++> directory of this distribution is relaxed
+ SWISH++ configuration that will index all words passed to it. This
+ configuration is needed for B<date test> because default configuration
+ doesn't recognize 2004-12-05 as date. Have in mind that your index size
+ might explode.
  =head1 SEE ALSO
  C<swish++> web site L<http://homepage.mac.com/pauljlucas/software/swish/>

 Legend:



Removed from v.5
 


changed lines


 
Added in v.16
 Legend:



Removed from v.5
 


changed lines


 
Added in v.16
-Removed from v.5
+Added in v.16

	ViewVC Help
Powered by ViewVC 1.1.26