--- trunk/bin/BackupPC_updatedb 2005/08/20 15:51:16 50 +++ trunk/bin/BackupPC_updatedb 2005/08/24 17:12:22 70 @@ -17,6 +17,8 @@ my $debug = 0; $|=1; +my $start_t = time(); + my $pidfile = new File::Pid; if (my $pid = $pidfile->running ) { @@ -36,11 +38,8 @@ my $TopDir = $bpc->TopDir(); my $beenThere = {}; -my $dsn = "dbi:SQLite:dbname=$TopDir/$Conf{SearchDB}"; -my $user = ''; - -# DEBUG option! -($dsn,$user) = qw/dbi:Pg:dbname=backuppc dpavlin/; +my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n"; +my $user = $Conf{SearchUser} || ''; my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 }); @@ -66,6 +65,7 @@ my $index = shift || return; my ($table,$col,$unique) = split(/_/, $index); $unique ||= ''; + $index =~ s/,/_/g; $dbh->do(qq{ create $unique index $index on $table($col) }); } @@ -95,11 +95,13 @@ num INTEGER NOT NULL, date integer NOT NULL, type CHAR(4) not null, - PRIMARY KEY(hostID, num) + shareID integer not null references shares(id), + size integer not null, + PRIMARY KEY(hostID, num, shareID) ); }); - do_index('backups_num_unique'); + #do_index('backups_hostid,num_unique'); $dbh->do(qq{ create table dvds ( @@ -114,10 +116,9 @@ create table files ( ID SERIAL PRIMARY KEY, shareID INTEGER NOT NULL references shares(id), - backupNum INTEGER NOT NULL references backups(num), + backupNum INTEGER NOT NULL, name VARCHAR(255) NOT NULL, path VARCHAR(255) NOT NULL, - fullpath VARCHAR(255) NOT NULL, date integer NOT NULL, type INTEGER NOT NULL, size INTEGER NOT NULL, @@ -156,7 +157,7 @@ } print " done...\n"; - eval { $dbh->commit; }; + $dbh->commit; } if ($opt{v}) { @@ -181,21 +182,21 @@ SELECT ID FROM hosts WHERE name=? }); -$sth->{backups_broj} = $dbh->prepare(qq{ +$sth->{backups_count} = $dbh->prepare(qq{ SELECT COUNT(*) FROM backups -WHERE hostID=? AND num=? +WHERE hostID=? AND num=? AND shareid=? }); $sth->{insert_backups} = $dbh->prepare(qq{ -INSERT INTO backups (hostID, num, date, type) -VALUES (?,?,?,?) +INSERT INTO backups (hostID, num, date, type, shareid, size) +VALUES (?,?,?,?,?,?) }); $sth->{insert_files} = $dbh->prepare(qq{ INSERT INTO files - (shareID, backupNum, name, path, fullpath, date, type, size) - VALUES (?,?,?,?,?,?,?,?) + (shareID, backupNum, name, path, date, type, size) + VALUES (?,?,?,?,?,?,?) }); sub fmt_time { @@ -222,13 +223,15 @@ $hostID = $dbh->last_insert_id(undef,undef,'hosts',undef); } - print("host ".$hosts->{$host_key}->{'host'}.": "); + print "host ".$hosts->{$host_key}->{'host'}.": "; # get backups for a host my @backups = $bpc->BackupInfoRead($hostname); - print scalar @backups, " increments\n"; + my $incs = scalar @backups; + print "$incs increments\n"; my $inc_nr = 0; + $beenThere = {}; foreach my $backup (@backups) { @@ -238,39 +241,51 @@ my $backupNum = $backup->{'num'}; my @backupShares = (); - print $hosts->{$host_key}->{'host'}, - "\t#$backupNum\t", $backup->{type} || '?', " ", - $backup->{nFilesNew} || '?', "/", $backup->{nFiles} || '?', - " files\n"; - - $sth->{backups_broj}->execute($hostID, $backupNum); - my ($broj) = $sth->{backups_broj}->fetchrow_array(); - next if ($broj > 0); - - $sth->{insert_backups}->execute( - $hostID, - $backupNum, - $backup->{'endTime'}, - $backup->{'type'} + printf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n", + $hosts->{$host_key}->{'host'}, + $inc_nr, $incs, $backupNum, + $backup->{type} || '?', + $backup->{nFilesNew} || '?', $backup->{nFiles} || '?', + strftime($t_fmt,localtime($backup->{startTime})), + fmt_time($backup->{endTime} - $backup->{startTime}) ); - $dbh->commit(); my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1); foreach my $share ($files->shareList($backupNum)) { my $t = time(); - print strftime($t_fmt,localtime())," ", $share; $shareID = getShareID($share, $hostID, $hostname); - my ($f, $nf, $d, $nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID); + $sth->{backups_count}->execute($hostID, $backupNum, $shareID); + my ($count) = $sth->{backups_count}->fetchrow_array(); + # skip if allready in database! + next if ($count > 0); + + # dump some log + print strftime($t_fmt,localtime())," ", $share; + + my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID); + + $sth->{insert_backups}->execute( + $hostID, + $backupNum, + $backup->{'endTime'}, + $backup->{'type'}, + $shareID, + $size, + ); + + print " commit"; + $dbh->commit(); + my $dur = (time() - $t) || 1; - printf(" %d/%d files %d/%d dirs [%.2f/s dur: %s]\n", + printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n", $nf, $f, $nd, $d, + ($size / 1024 / 1024), ( ($f+$d) / $dur ), fmt_time($dur) ); - $dbh->commit(); } } @@ -279,6 +294,8 @@ $dbh->commit(); $dbh->disconnect(); +print "total duration: ",fmt_time(time() - $start_t),"\n"; + $pidfile->remove; sub getShareID() { @@ -321,12 +338,11 @@ SELECT 1 FROM files WHERE shareID = ? and path = ? and - name = ? and date = ? and size = ? }); - my @param = ($shareID,$path,$name,$date,$size); + my @param = ($shareID,$path,$date,$size); $sth->{file_in_db}->execute(@param); my $rows = $sth->{file_in_db}->rows; print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3); @@ -347,7 +363,7 @@ print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1); - my ($nr_files, $new_files, $nr_dirs, $new_dirs) = (0,0,0,0); + my ($nr_files, $new_files, $nr_dirs, $new_dirs, $size) = (0,0,0,0,0); { # scope my @stack; @@ -357,13 +373,12 @@ # first, add all the entries in current directory foreach my $path_key (keys %{$filesInBackup}) { + print STDERR "# file ",Dumper($filesInBackup->{$path_key}),"\n" if ($debug >= 3); my @data = ( $shareID, $backupNum, $path_key, $filesInBackup->{$path_key}->{'relPath'}, - $filesInBackup->{$path_key}->{'fullPath'}, - # $filesInBackup->{$path_key}->{'sharePathM'}, $filesInBackup->{$path_key}->{'mtime'}, $filesInBackup->{$path_key}->{'type'}, $filesInBackup->{$path_key}->{'size'} @@ -377,17 +392,18 @@ $filesInBackup->{$path_key}->{'size'} )); - - if (! defined($beenThere->{$key}) && ! found_in_db($key, @data)) { + my $found; + if (! defined($beenThere->{$key}) && ! ($found = found_in_db($key, @data)) ) { print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2); if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) { - $new_dirs++; + $new_dirs++ unless ($found); print STDERR " dir\n" if ($debug >= 2); } else { - $new_files++; + $new_files++ unless ($found); print STDERR " file\n" if ($debug >= 2); } + $size += $filesInBackup->{$path_key}->{'size'} || 0; } if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) { @@ -412,15 +428,16 @@ print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2); while ( my $dir = shift @stack ) { - my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID); + my ($f,$nf,$d,$nd, $s) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID); print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1); $nr_files += $f; $new_files += $nf; $nr_dirs += $d; $new_dirs += $nd; + $size += $s; } } - return ($nr_files, $new_files, $nr_dirs, $new_dirs); + return ($nr_files, $new_files, $nr_dirs, $new_dirs, $size); }