--- trunk/bin/BackupPC_updatedb 2005/08/24 17:12:22 70 +++ trunk/bin/BackupPC_updatedb 2005/08/31 11:05:26 104 @@ -13,6 +13,7 @@ use POSIX qw/strftime/; use constant BPC_FTYPE_DIR => 5; +use constant EST_CHUNK => 100000; my $debug = 0; $|=1; @@ -40,26 +41,203 @@ my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n"; my $user = $Conf{SearchUser} || ''; +my $index_path = $Conf{HyperEstraierIndex}; +$index_path = $TopDir . '/' . $index_path; +$index_path =~ s#//#/#g; +if ($index_path) { + use HyperEstraier; +} + my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 }); my %opt; -if ( !getopts("cdm:v:", \%opt ) ) { +if ( !getopts("cdm:v:i", \%opt ) ) { print STDERR <sync(); + $hest_db->close(); + } + exit(0); +} + +$SIG{'INT'} = \&signal; +$SIG{'QUIT'} = \&signal; + +sub hest_update { + + my ($host_id, $share_id, $num) = @_; + + print curr_time," updating HyperEstraier:"; + + my $t = time(); + + my $offset = 0; + my $added = 0; + + print " opening index $index_path"; + $hest_db = HyperEstraier::Database->new(); + $hest_db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT); + + print " increment is " . EST_CHUNK . " files"; + + my $results = 0; + + do { + + my $where = ''; + my @data; + if ($host_id && $share_id && $num) { + $where = qq{ + WHERE + hosts.id = ? AND + shares.id = ? AND + files.backupnum = ? + }; + @data = ( $host_id, $share_id, $num ); + } + + my $limit = sprintf('LIMIT '.EST_CHUNK.' OFFSET %d', $offset); + + my $sth = $dbh->prepare(qq{ + SELECT + files.id AS fid, + hosts.name AS hname, + shares.name AS sname, + -- shares.share AS sharename, + files.backupnum AS backupnum, + -- files.name AS filename, + files.path AS filepath, + files.date AS date, + files.type AS type, + files.size AS size, + files.shareid AS shareid, + backups.date AS backup_date + FROM files + INNER JOIN shares ON files.shareID=shares.ID + INNER JOIN hosts ON hosts.ID = shares.hostID + INNER JOIN backups ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID + $where + $limit + }); + + $sth->execute(@data); + $results = $sth->rows; + + if ($results == 0) { + print " - no more files\n"; + last; + } + + sub fmt_date { + my $t = shift || return; + my $iso = BackupPC::Lib::timeStamp($t); + $iso =~ s/\s/T/; + return $iso; + } + + while (my $row = $sth->fetchrow_hashref()) { + + my $fid = $row->{'fid'} || die "no fid?"; + my $uri = 'file:///' . $fid; + + my $id = $hest_db->uri_to_id($uri); + next unless ($id == -1); + + # create a document object + my $doc = HyperEstraier::Document->new; + + # add attributes to the document object + $doc->add_attr('@uri', $uri); + + foreach my $c (@{ $sth->{NAME} }) { + $doc->add_attr($c, $row->{$c}) if ($row->{$c}); + } + + #$doc->add_attr('@cdate', fmt_date($row->{'date'})); + + # add the body text to the document object + my $path = $row->{'filepath'}; + $doc->add_text($path); + $path =~ s/(.)/$1 /g; + $doc->add_hidden_text($path); + + print STDERR $doc->dump_draft,"\n" if ($debug > 1); + + # register the document object to the database + $hest_db->put_doc($doc, $HyperEstraier::Database::PDCLEAN); + $added++; + } + + print " $added"; + $hest_db->sync(); + + $offset += EST_CHUNK; + + } while ($results == EST_CHUNK); + + print ", close"; + $hest_db->close(); + + my $dur = (time() - $t) || 1; + printf(" [%.2f/s dur: %s]\n", + ( $added / $dur ), + fmt_time($dur) + ); +} + +#---- /subs ---- + + +## update index ## +if (($opt{i} || ($index_path && ! -e $index_path)) && !$opt{c}) { + # update all + print "force update of HyperEstraier index "; + print "importing existing data" unless (-e $index_path); + print "by -i flag" if ($opt{i}); + print "\n"; + hest_update(); +} + +## create tables ## if ($opt{c}) { sub do_index { my $index = shift || return; @@ -149,6 +327,7 @@ } +## delete data before inseting ## if ($opt{d}) { print "deleting "; foreach my $table (qw(files dvds backups shares hosts)) { @@ -160,12 +339,7 @@ $dbh->commit; } -if ($opt{v}) { - print "Debug level at $opt{v}\n"; - $debug = $opt{v}; -} - -#################################INSERT VALUES############################# +## insert new values ## # get hosts $hosts = $bpc->HostInfoRead(); @@ -199,15 +373,6 @@ VALUES (?,?,?,?,?,?,?) }); -sub fmt_time { - my $t = shift || return; - my $out = ""; - my ($ss,$mm,$hh) = gmtime($t); - $out .= "${hh}h" if ($hh); - $out .= sprintf("%02d:%02d", $mm,$ss); - return $out; -} - foreach my $host_key (keys %{$hosts}) { my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key"; @@ -263,7 +428,7 @@ next if ($count > 0); # dump some log - print strftime($t_fmt,localtime())," ", $share; + print curr_time," ", $share; my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID); @@ -286,6 +451,8 @@ ( ($f+$d) / $dur ), fmt_time($dur) ); + + hest_update($hostID, $shareID, $backupNum); } } @@ -330,7 +497,7 @@ my @data = @_; shift @data; - my ($key, $shareID,undef,$name,$path,undef,$date,undef,$size) = @_; + my ($key, $shareID,undef,$name,$path,$date,undef,$size) = @_; return $beenThere->{$key} if (defined($beenThere->{$key})); @@ -340,12 +507,13 @@ path = ? and date = ? and size = ? + LIMIT 1 }); my @param = ($shareID,$path,$date,$size); $sth->{file_in_db}->execute(@param); my $rows = $sth->{file_in_db}->rows; - print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3); + print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3); $beenThere->{$key}++;