Grep/bin/import.pl

#!/usr/bin/perl

# helper script to import external data sources into local index and full text index

use strict;

use lib 'lib';

use Jifty;
use Grep::Search;
use Module::Pluggable search_path => 'Grep::Import', sub_name => 'importers', require => 1;
use Data::Dump qw/dump/;

$|=1;

BEGIN { Jifty->new; };

my @importers = __PACKAGE__->importers();

Jifty->log->debug("Found import plugins: ", join(", ", @importers) );

my $system_user = Grep::CurrentUser->superuser;
my $search = Grep::Search->new();


foreach my $importer ( @importers ) {

        if ( $importer->can('import') ) {
                Jifty->log->info("importing with $importer");
                $importer->import( $search );
        } else {
                Jifty->log->error("importer $importer doesn't implement import");
        }
}


__END__
my $coll = Grep::Model::ItemCollection->new( results_are_readable => 1, current_user => $system_user );
$coll->unlimit;

print "indexing ", $coll->count, " items ", $keep_duplicates ? "" : "removing duplicates ";

exit;


my ( $total ) = ( 0 );

while ( my $i = $coll->next ) {

        print $i->id;

        $search->add( $i, $i->in_feed->owner->id );
        print ' ';
        $total++;
}

print "$total records indexed", $duplicates ? "($duplicates duplicates)" : "", "\n";

$search->finish;
1	dpavlin	154	#!/usr/bin/perl
2
3			# helper script to import external data sources into local index and full text index
4
5			use strict;
6
7			use lib 'lib';
8
9			use Jifty;
10			use Grep::Search;
11			use Module::Pluggable search_path => 'Grep::Import', sub_name => 'importers', require => 1;
12			use Data::Dump qw/dump/;
13
14			$\|=1;
15
16			BEGIN { Jifty->new; };
17
18			my @importers = __PACKAGE__->importers();
19
20			Jifty->log->debug("Found import plugins: ", join(", ", @importers) );
21
22			my $system_user = Grep::CurrentUser->superuser;
23			my $search = Grep::Search->new();
24
25
26			foreach my $importer ( @importers ) {
27
28			if ( $importer->can('import') ) {
29			Jifty->log->info("importing with $importer");
30			$importer->import( $search );
31			} else {
32			Jifty->log->error("importer $importer doesn't implement import");
33			}
34			}
35
36
37			__END__
38			my $coll = Grep::Model::ItemCollection->new( results_are_readable => 1, current_user => $system_user );
39			$coll->unlimit;
40
41			print "indexing ", $coll->count, " items ", $keep_duplicates ? "" : "removing duplicates ";
42
43			exit;
44
45
46			my ( $total ) = ( 0 );
47
48			while ( my $i = $coll->next ) {
49
50			print $i->id;
51
52			$search->add( $i, $i->in_feed->owner->id );
53			print ' ';
54			$total++;
55			}
56
57			print "$total records indexed", $duplicates ? "($duplicates duplicates)" : "", "\n";
58
59			$search->finish;