--- trunk/bin/BackupPC_updatedb 2005/08/28 18:02:58 92 +++ trunk/bin/BackupPC_updatedb 2005/08/31 11:05:26 104 @@ -13,6 +13,7 @@ use POSIX qw/strftime/; use constant BPC_FTYPE_DIR => 5; +use constant EST_CHUNK => 100000; my $debug = 0; $|=1; @@ -43,6 +44,9 @@ my $index_path = $Conf{HyperEstraierIndex}; $index_path = $TopDir . '/' . $index_path; $index_path =~ s#//#/#g; +if ($index_path) { + use HyperEstraier; +} my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 }); @@ -85,128 +89,136 @@ my $hest_db; +sub signal { + my($sig) = @_; + if ($hest_db) { + print "\nCaught a SIG$sig--syncing database and shutting down\n"; + $hest_db->sync(); + $hest_db->close(); + } + exit(0); +} + +$SIG{'INT'} = \&signal; +$SIG{'QUIT'} = \&signal; + sub hest_update { my ($host_id, $share_id, $num) = @_; - print curr_time," updating HyperEstraier: select files"; + print curr_time," updating HyperEstraier:"; my $t = time(); - my $where = ''; - if ($host_id && $share_id && $num) { - $where = qq{ - WHERE - hosts.id = ? AND - shares.id = ? AND - files.backupnum = ? - }; - } - - my $sth = $dbh->prepare(qq{ - SELECT - files.id AS fid, - hosts.name AS hname, - shares.name AS sname, - -- shares.share AS sharename, - files.backupnum AS backupnum, - -- files.name AS filename, - files.path AS filepath, - files.date AS date, - files.type AS type, - files.size AS size, - files.shareid AS shareid, - backups.date AS backup_date - FROM files - INNER JOIN shares ON files.shareID=shares.ID - INNER JOIN hosts ON hosts.ID = shares.hostID - INNER JOIN backups ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID - $where - }); - - $sth->execute(@_); - my $results = $sth->rows; - - if ($results == 0) { - print " - no files, skipping\n"; - return; - } - - my $dot = int($results / 15) || 1; + my $offset = 0; + my $added = 0; - print " $results ($dot/#)"; + print " opening index $index_path"; + $hest_db = HyperEstraier::Database->new(); + $hest_db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT); + + print " increment is " . EST_CHUNK . " files"; + + my $results = 0; + + do { + + my $where = ''; + my @data; + if ($host_id && $share_id && $num) { + $where = qq{ + WHERE + hosts.id = ? AND + shares.id = ? AND + files.backupnum = ? + }; + @data = ( $host_id, $share_id, $num ); + } - sub fmt_date { - my $t = shift || return; - my $iso = BackupPC::Lib::timeStamp($t); - $iso =~ s/\s/T/; - return $iso; - } + my $limit = sprintf('LIMIT '.EST_CHUNK.' OFFSET %d', $offset); - my $max = int($results / $dot); + my $sth = $dbh->prepare(qq{ + SELECT + files.id AS fid, + hosts.name AS hname, + shares.name AS sname, + -- shares.share AS sharename, + files.backupnum AS backupnum, + -- files.name AS filename, + files.path AS filepath, + files.date AS date, + files.type AS type, + files.size AS size, + files.shareid AS shareid, + backups.date AS backup_date + FROM files + INNER JOIN shares ON files.shareID=shares.ID + INNER JOIN hosts ON hosts.ID = shares.hostID + INNER JOIN backups ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID + $where + $limit + }); + + $sth->execute(@data); + $results = $sth->rows; + + if ($results == 0) { + print " - no more files\n"; + last; + } - print ", opening index $index_path..."; - use HyperEstraier; - my $db = HyperEstraier::Database->new(); + sub fmt_date { + my $t = shift || return; + my $iso = BackupPC::Lib::timeStamp($t); + $iso =~ s/\s/T/; + return $iso; + } -# unless ($hest_db) { -# print " open reader"; -# $hest_db = HyperEstraier::Database->new(); -# -# } + while (my $row = $sth->fetchrow_hashref()) { + my $fid = $row->{'fid'} || die "no fid?"; + my $uri = 'file:///' . $fid; - $db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT); + my $id = $hest_db->uri_to_id($uri); + next unless ($id == -1); - my $added = 0; + # create a document object + my $doc = HyperEstraier::Document->new; - while (my $row = $sth->fetchrow_hashref()) { + # add attributes to the document object + $doc->add_attr('@uri', $uri); - my $fid = $row->{'fid'} || die "no fid?"; - my $uri = 'file:///' . $fid; + foreach my $c (@{ $sth->{NAME} }) { + $doc->add_attr($c, $row->{$c}) if ($row->{$c}); + } - my $id = $db->uri_to_id($uri); - next unless ($id == -1); + #$doc->add_attr('@cdate', fmt_date($row->{'date'})); - # create a document object - my $doc = HyperEstraier::Document->new; + # add the body text to the document object + my $path = $row->{'filepath'}; + $doc->add_text($path); + $path =~ s/(.)/$1 /g; + $doc->add_hidden_text($path); - # add attributes to the document object - $doc->add_attr('@uri', $uri); + print STDERR $doc->dump_draft,"\n" if ($debug > 1); - foreach my $c (@{ $sth->{NAME} }) { - $doc->add_attr($c, $row->{$c}) if ($row->{$c}); + # register the document object to the database + $hest_db->put_doc($doc, $HyperEstraier::Database::PDCLEAN); + $added++; } - #$doc->add_attr('@cdate', fmt_date($row->{'date'})); + print " $added"; + $hest_db->sync(); - # add the body text to the document object - my $path = $row->{'filepath'}; - $doc->add_text($path); - $path =~ s/(.)/$1 /g; - $doc->add_hidden_text($path); - - print STDERR $doc->dump_draft,"\n" if ($debug > 1); - - # register the document object to the database - $db->put_doc($doc, $HyperEstraier::Database::PDCLEAN); - - $added++; - if ($added % $dot == 0) { - print "$max "; - $max--; - } + $offset += EST_CHUNK; - } + } while ($results == EST_CHUNK); - print "sync $added new files"; - $db->sync(); print ", close"; - $db->close(); + $hest_db->close(); my $dur = (time() - $t) || 1; - printf(" [%.2f/s new %.2f/s dur: %s]\n", - ( $results / $dur ), + printf(" [%.2f/s dur: %s]\n", ( $added / $dur ), fmt_time($dur) ); @@ -216,7 +228,7 @@ ## update index ## -if ($opt{i} || ($index_path && ! -e $index_path)) { +if (($opt{i} || ($index_path && ! -e $index_path)) && !$opt{c}) { # update all print "force update of HyperEstraier index "; print "importing existing data" unless (-e $index_path);