Grep/bin/reindex.pl

#!/usr/bin/perl

# helper script to re-index full text index

use strict;

use lib 'lib';

use Jifty;
use Grep::Search;
use Data::Dump qw/dump/;
use Text::DeDuper;
use Encode;

my $remove_duplicates = 1;

$|=1;

BEGIN { Jifty->new; };

my $system_user = Grep::CurrentUser->superuser;
my $coll = Grep::Model::ItemCollection->new( results_are_readable => 1, current_user => $system_user );
$coll->unlimit;

print "indexing ", $coll->count, " items ";

my $search = Grep::Search->new();
my $deduper = Text::DeDuper->new();

my ( $total, $duplicates ) = ( 0, 0 );

while ( my $i = $coll->next ) {

        print $i->id;

        if ( $remove_duplicates ) {

                my $c = encode('utf-8', $i->content);

                my @s = sort $deduper->find_similar( $c );
                if ( @s ) {
                        #warn " similar = ",dump( @s );

                        foreach my $id ( @s ) {
                                next if $id == $i->id;  # keep current
                                my $si = Grep::Model::Item->new();
                                $si->load( $id ) or die "can't find similar item $id";
                                print " -$id-";
                                $si->delete;
                                $duplicates++;
                        }
                }

                $deduper->add_doc( $i->id, $c );
        }

        $search->add( $i, $i->in_feed->owner->id );
        print ' ';
        $total++;
}

print "$total records indexed", $remove_duplicates ? " ($duplicates duplicates)" : "", "\n";

$search->finish;
1	#!/usr/bin/perl
2
3	# helper script to re-index full text index
4
5	use strict;
6
7	use lib 'lib';
8
9	use Jifty;
10	use Grep::Search;
11	use Data::Dump qw/dump/;
12	use Text::DeDuper;
13	use Encode;
14
15	my $remove_duplicates = 1;
16
17	$\|=1;
18
19	BEGIN { Jifty->new; };
20
21	my $system_user = Grep::CurrentUser->superuser;
22	my $coll = Grep::Model::ItemCollection->new( results_are_readable => 1, current_user => $system_user );
23	$coll->unlimit;
24
25	print "indexing ", $coll->count, " items ";
26
27	my $search = Grep::Search->new();
28	my $deduper = Text::DeDuper->new();
29
30	my ( $total, $duplicates ) = ( 0, 0 );
31
32	while ( my $i = $coll->next ) {
33
34	print $i->id;
35
36	if ( $remove_duplicates ) {
37
38	my $c = encode('utf-8', $i->content);
39
40	my @s = sort $deduper->find_similar( $c );
41	if ( @s ) {
42	#warn " similar = ",dump( @s );
43
44	foreach my $id ( @s ) {
45	next if $id == $i->id; # keep current
46	my $si = Grep::Model::Item->new();
47	$si->load( $id ) or die "can't find similar item $id";
48	print " -$id-";
49	$si->delete;
50	$duplicates++;
51	}
52	}
53
54	$deduper->add_doc( $i->id, $c );
55	}
56
57	$search->add( $i, $i->in_feed->owner->id );
58	print ' ';
59	$total++;
60	}
61
62	print "$total records indexed", $remove_duplicates ? " ($duplicates duplicates)" : "", "\n";
63
64	$search->finish;