--- trunk/bin/BackupPC_updatedb 2005/08/20 15:51:16 50 +++ trunk/bin/BackupPC_updatedb 2005/08/28 12:35:59 86 @@ -17,6 +17,8 @@ my $debug = 0; $|=1; +my $start_t = time(); + my $pidfile = new File::Pid; if (my $pid = $pidfile->running ) { @@ -36,29 +38,150 @@ my $TopDir = $bpc->TopDir(); my $beenThere = {}; -my $dsn = "dbi:SQLite:dbname=$TopDir/$Conf{SearchDB}"; -my $user = ''; - -# DEBUG option! -($dsn,$user) = qw/dbi:Pg:dbname=backuppc dpavlin/; +my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n"; +my $user = $Conf{SearchUser} || ''; +my $index_path = $Conf{HyperEstraierIndex}; my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 }); my %opt; -if ( !getopts("cdm:v:", \%opt ) ) { +if ( !getopts("cdm:v:i", \%opt ) ) { print STDERR <prepare(qq{ + SELECT + files.id AS fid, + hosts.name AS hname, + shares.name AS sname, + -- shares.share AS sharename, + files.backupnum AS backupnum, + -- files.name AS filename, + files.path AS filepath, + files.date AS date, + files.type AS filetype, + files.size AS size, + files.shareid AS shareid, + backups.date AS backup_date + FROM files + INNER JOIN shares ON files.shareID=shares.ID + INNER JOIN hosts ON hosts.ID = shares.hostID + INNER JOIN backups ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID + }); + + $sth->execute(); + my $results = $sth->rows; + + my $dot = int($results / 15); + + print " $results ($dot/#)"; + + sub fmt_date { + my $t = shift || return; + my $iso = BackupPC::Lib::timeStamp($t); + $iso =~ s/\s/T/; + return $iso; + } + + my $i = 0; + my $max = int($results / $dot); + + $index_path = $TopDir . '/' . $index_path; + $index_path =~ s#//#/#g; + + print " index $index_path..."; + use HyperEstraier; + my $db = HyperEstraier::Database->new(); + $db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT); + + + while (my $row = $sth->fetchrow_hashref()) { + + # create a document object + my $doc = HyperEstraier::Document->new; + + # add attributes to the document object + $doc->add_attr('@uri', 'file:///' . $row->{'fid'}); + + foreach my $c (@{ $sth->{NAME} }) { + $doc->add_attr($c, $row->{$c}) if ($row->{$c}); + } + + #$doc->add_attr('@cdate', fmt_date($row->{'date'})); + + # add the body text to the document object + my $path = $row->{'filepath'}; + $doc->add_text($path); + $path =~ s/(.)/$1 /g; + $doc->add_hidden_text($path); + + print STDERR $doc->dump_draft,"\n" if ($debug > 1); + + # register the document object to the database + $db->put_doc($doc, $HyperEstraier::Database::PDCLEAN); + + $i++; + if ($i % $dot == 0) { + print "$max "; + $max--; + } + + } + + print "sync"; + $db->sync(); + print " close"; + $db->close(); + + my $dur = (time() - $t) || 1; + printf(" [%.2f/s dur: %s]\n", + ( $results / $dur ), + fmt_time($dur) + ); + + exit; +} + ###################################create tables############################3 if ($opt{c}) { @@ -66,6 +189,7 @@ my $index = shift || return; my ($table,$col,$unique) = split(/_/, $index); $unique ||= ''; + $index =~ s/,/_/g; $dbh->do(qq{ create $unique index $index on $table($col) }); } @@ -95,11 +219,13 @@ num INTEGER NOT NULL, date integer NOT NULL, type CHAR(4) not null, - PRIMARY KEY(hostID, num) + shareID integer not null references shares(id), + size integer not null, + PRIMARY KEY(hostID, num, shareID) ); }); - do_index('backups_num_unique'); + #do_index('backups_hostid,num_unique'); $dbh->do(qq{ create table dvds ( @@ -114,10 +240,9 @@ create table files ( ID SERIAL PRIMARY KEY, shareID INTEGER NOT NULL references shares(id), - backupNum INTEGER NOT NULL references backups(num), + backupNum INTEGER NOT NULL, name VARCHAR(255) NOT NULL, path VARCHAR(255) NOT NULL, - fullpath VARCHAR(255) NOT NULL, date integer NOT NULL, type INTEGER NOT NULL, size INTEGER NOT NULL, @@ -156,12 +281,7 @@ } print " done...\n"; - eval { $dbh->commit; }; -} - -if ($opt{v}) { - print "Debug level at $opt{v}\n"; - $debug = $opt{v}; + $dbh->commit; } #################################INSERT VALUES############################# @@ -181,32 +301,23 @@ SELECT ID FROM hosts WHERE name=? }); -$sth->{backups_broj} = $dbh->prepare(qq{ +$sth->{backups_count} = $dbh->prepare(qq{ SELECT COUNT(*) FROM backups -WHERE hostID=? AND num=? +WHERE hostID=? AND num=? AND shareid=? }); $sth->{insert_backups} = $dbh->prepare(qq{ -INSERT INTO backups (hostID, num, date, type) -VALUES (?,?,?,?) +INSERT INTO backups (hostID, num, date, type, shareid, size) +VALUES (?,?,?,?,?,?) }); $sth->{insert_files} = $dbh->prepare(qq{ INSERT INTO files - (shareID, backupNum, name, path, fullpath, date, type, size) - VALUES (?,?,?,?,?,?,?,?) + (shareID, backupNum, name, path, date, type, size) + VALUES (?,?,?,?,?,?,?) }); -sub fmt_time { - my $t = shift || return; - my $out = ""; - my ($ss,$mm,$hh) = gmtime($t); - $out .= "${hh}h" if ($hh); - $out .= sprintf("%02d:%02d", $mm,$ss); - return $out; -} - foreach my $host_key (keys %{$hosts}) { my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key"; @@ -222,13 +333,15 @@ $hostID = $dbh->last_insert_id(undef,undef,'hosts',undef); } - print("host ".$hosts->{$host_key}->{'host'}.": "); + print "host ".$hosts->{$host_key}->{'host'}.": "; # get backups for a host my @backups = $bpc->BackupInfoRead($hostname); - print scalar @backups, " increments\n"; + my $incs = scalar @backups; + print "$incs increments\n"; my $inc_nr = 0; + $beenThere = {}; foreach my $backup (@backups) { @@ -238,39 +351,51 @@ my $backupNum = $backup->{'num'}; my @backupShares = (); - print $hosts->{$host_key}->{'host'}, - "\t#$backupNum\t", $backup->{type} || '?', " ", - $backup->{nFilesNew} || '?', "/", $backup->{nFiles} || '?', - " files\n"; - - $sth->{backups_broj}->execute($hostID, $backupNum); - my ($broj) = $sth->{backups_broj}->fetchrow_array(); - next if ($broj > 0); - - $sth->{insert_backups}->execute( - $hostID, - $backupNum, - $backup->{'endTime'}, - $backup->{'type'} + printf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n", + $hosts->{$host_key}->{'host'}, + $inc_nr, $incs, $backupNum, + $backup->{type} || '?', + $backup->{nFilesNew} || '?', $backup->{nFiles} || '?', + strftime($t_fmt,localtime($backup->{startTime})), + fmt_time($backup->{endTime} - $backup->{startTime}) ); - $dbh->commit(); my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1); foreach my $share ($files->shareList($backupNum)) { my $t = time(); - print strftime($t_fmt,localtime())," ", $share; $shareID = getShareID($share, $hostID, $hostname); - my ($f, $nf, $d, $nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID); + $sth->{backups_count}->execute($hostID, $backupNum, $shareID); + my ($count) = $sth->{backups_count}->fetchrow_array(); + # skip if allready in database! + next if ($count > 0); + + # dump some log + print curr_time," ", $share; + + my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID); + + $sth->{insert_backups}->execute( + $hostID, + $backupNum, + $backup->{'endTime'}, + $backup->{'type'}, + $shareID, + $size, + ); + + print " commit"; + $dbh->commit(); + my $dur = (time() - $t) || 1; - printf(" %d/%d files %d/%d dirs [%.2f/s dur: %s]\n", + printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n", $nf, $f, $nd, $d, + ($size / 1024 / 1024), ( ($f+$d) / $dur ), fmt_time($dur) ); - $dbh->commit(); } } @@ -279,6 +404,8 @@ $dbh->commit(); $dbh->disconnect(); +print "total duration: ",fmt_time(time() - $start_t),"\n"; + $pidfile->remove; sub getShareID() { @@ -313,7 +440,7 @@ my @data = @_; shift @data; - my ($key, $shareID,undef,$name,$path,undef,$date,undef,$size) = @_; + my ($key, $shareID,undef,$name,$path,$date,undef,$size) = @_; return $beenThere->{$key} if (defined($beenThere->{$key})); @@ -321,15 +448,15 @@ SELECT 1 FROM files WHERE shareID = ? and path = ? and - name = ? and date = ? and size = ? + LIMIT 1 }); - my @param = ($shareID,$path,$name,$date,$size); + my @param = ($shareID,$path,$date,$size); $sth->{file_in_db}->execute(@param); my $rows = $sth->{file_in_db}->rows; - print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3); + print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3); $beenThere->{$key}++; @@ -347,7 +474,7 @@ print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1); - my ($nr_files, $new_files, $nr_dirs, $new_dirs) = (0,0,0,0); + my ($nr_files, $new_files, $nr_dirs, $new_dirs, $size) = (0,0,0,0,0); { # scope my @stack; @@ -357,13 +484,12 @@ # first, add all the entries in current directory foreach my $path_key (keys %{$filesInBackup}) { + print STDERR "# file ",Dumper($filesInBackup->{$path_key}),"\n" if ($debug >= 3); my @data = ( $shareID, $backupNum, $path_key, $filesInBackup->{$path_key}->{'relPath'}, - $filesInBackup->{$path_key}->{'fullPath'}, - # $filesInBackup->{$path_key}->{'sharePathM'}, $filesInBackup->{$path_key}->{'mtime'}, $filesInBackup->{$path_key}->{'type'}, $filesInBackup->{$path_key}->{'size'} @@ -377,17 +503,18 @@ $filesInBackup->{$path_key}->{'size'} )); - - if (! defined($beenThere->{$key}) && ! found_in_db($key, @data)) { + my $found; + if (! defined($beenThere->{$key}) && ! ($found = found_in_db($key, @data)) ) { print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2); if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) { - $new_dirs++; + $new_dirs++ unless ($found); print STDERR " dir\n" if ($debug >= 2); } else { - $new_files++; + $new_files++ unless ($found); print STDERR " file\n" if ($debug >= 2); } + $size += $filesInBackup->{$path_key}->{'size'} || 0; } if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) { @@ -412,15 +539,16 @@ print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2); while ( my $dir = shift @stack ) { - my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID); + my ($f,$nf,$d,$nd, $s) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID); print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1); $nr_files += $f; $new_files += $nf; $nr_dirs += $d; $new_dirs += $nd; + $size += $s; } } - return ($nr_files, $new_files, $nr_dirs, $new_dirs); + return ($nr_files, $new_files, $nr_dirs, $new_dirs, $size); }