/[Grep]/bin/reindex.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /bin/reindex.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 129 by dpavlin, Sun Apr 29 11:37:28 2007 UTC revision 193 by dpavlin, Fri May 23 22:31:37 2008 UTC
# Line 11  use Grep::Search; Line 11  use Grep::Search;
11  use Data::Dump qw/dump/;  use Data::Dump qw/dump/;
12  use Text::DeDuper;  use Text::DeDuper;
13  use Encode;  use Encode;
14    use Getopt::Long;
15    
16  my $remove_duplicates = 1;  my $keep_duplicates = 1;
17    
18    GetOptions(
19            'duplicates!' => $keep_duplicates,
20    );
21    
22  $|=1;  $|=1;
23    
# Line 22  my $system_user = Grep::CurrentUser->sup Line 27  my $system_user = Grep::CurrentUser->sup
27  my $coll = Grep::Model::ItemCollection->new( results_are_readable => 1, current_user => $system_user );  my $coll = Grep::Model::ItemCollection->new( results_are_readable => 1, current_user => $system_user );
28  $coll->unlimit;  $coll->unlimit;
29    
30  print "indexing ", $coll->count, " items ";  Jifty->log->info( "indexing ", $coll->count, " items ", $keep_duplicates ? "" : "removing duplicates " );
31    
32  my $search = Grep::Search->new();  my $search = Grep::Search->new({ create => 1 });
33  my $deduper = Text::DeDuper->new();  my $deduper = Text::DeDuper->new();
34    
35  my ( $total, $duplicates ) = ( 0, 0 );  my ( $total, $duplicates ) = ( 0, 0 );
# Line 33  while ( my $i = $coll->next ) { Line 38  while ( my $i = $coll->next ) {
38    
39          print $i->id;          print $i->id;
40    
41          if ( $remove_duplicates ) {          if ( ! $keep_duplicates ) {
42    
43                  my $c = encode('utf-8', $i->content);                  my $c = encode('utf-8', $i->content);
44    
# Line 48  while ( my $i = $coll->next ) { Line 53  while ( my $i = $coll->next ) {
53                                  print " -$id-";                                  print " -$id-";
54                                  $si->delete;                                  $si->delete;
55                                  $duplicates++;                                  $duplicates++;
56                                    $search->invindexer->delete_by_term( 'id', $id );
57                          }                          }
58                  }                  }
59    
# Line 59  while ( my $i = $coll->next ) { Line 65  while ( my $i = $coll->next ) {
65          $total++;          $total++;
66  }  }
67    
68  print "$total records indexed", $remove_duplicates ? " ($duplicates duplicates)" : "", "\n";  Jifty->log->info( "$total records indexed", $duplicates ? " ($duplicates duplicates)" : "" );
69    
70  $search->finish;  $search->finish;
Legend:
Removed from v.129  
changed lines
  Added in v.193
  ViewVC Help
Powered by ViewVC 1.1.26