/[BackupPC]/trunk/bin/BackupPC_tarIncCreate
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/bin/BackupPC_tarIncCreate

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 235 by dpavlin, Thu Nov 10 15:07:10 2005 UTC revision 254 by dpavlin, Mon Dec 12 16:07:27 2005 UTC
# Line 86  my %opts; Line 86  my %opts;
86  my $in_backup_increment;  my $in_backup_increment;
87    
88    
89  if ( !getopts("th:n:p:r:s:b:w:vd", \%opts) ) {  if ( !getopts("th:n:p:r:s:b:w:vdf", \%opts) ) {
90      print STDERR <<EOF;      print STDERR <<EOF;
91  usage: $0 [options]  usage: $0 [options]
92    Required options:    Required options:
# Line 102  usage: $0 [options] Line 102  usage: $0 [options]
102       -p pathAdd      new path prefix       -p pathAdd      new path prefix
103       -b BLOCKS       BLOCKS x 512 bytes per record (default 20; same as tar)       -b BLOCKS       BLOCKS x 512 bytes per record (default 20; same as tar)
104       -w writeBufSz   write buffer size (default 1048576 = 1MB)       -w writeBufSz   write buffer size (default 1048576 = 1MB)
105         -f              overwrite existing parts
106       -v              verbose output       -v              verbose output
107       -d              debug output       -d              debug output
108  EOF  EOF
# Line 109  EOF Line 110  EOF
110  }  }
111    
112  if ( $opts{h} !~ /^([\w\.\s-]+)$/ ) {  if ( $opts{h} !~ /^([\w\.\s-]+)$/ ) {
113      print(STDERR "$0: bad host name '$opts{h}'\n");      die "$0: bad host name '$opts{h}'\n";
     exit(1);  
114  }  }
115  my $Host = $opts{h};  my $Host = $opts{h};
116    
117  if ( $opts{n} !~ /^(-?\d+)$/ ) {  if ( $opts{n} !~ /^(-?\d+)$/ ) {
118      print(STDERR "$0: bad dump number '$opts{n}'\n");      die "$0: bad dump number '$opts{n}'\n";
     exit(1);  
119  }  }
120  my $Num = $opts{n};  my $Num = $opts{n};
121    
# Line 132  my $DirCnt = 0; Line 131  my $DirCnt = 0;
131  my $SpecialCnt = 0;  my $SpecialCnt = 0;
132  my $ErrorCnt = 0;  my $ErrorCnt = 0;
133  my $current_tar_size = 0;  my $current_tar_size = 0;
134    my $total_increment_size = 0;
135    
136  my $i;  my $i;
137  $Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 );  $Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 );
# Line 139  for ( $i = 0 ; $i < @Backups ; $i++ ) { Line 139  for ( $i = 0 ; $i < @Backups ; $i++ ) {
139      last if ( $Backups[$i]{num} == $Num );      last if ( $Backups[$i]{num} == $Num );
140  }  }
141  if ( $i >= @Backups ) {  if ( $i >= @Backups ) {
142      print(STDERR "$0: bad backup number $Num for host $Host\n");      die "$0: bad backup number $Num for host $Host\n";
     exit(1);  
143  }  }
144    
145  my $PathRemove = $1 if ( $opts{r} =~ /(.+)/ );  my $PathRemove = $1 if ( $opts{r} =~ /(.+)/ );
146  my $PathAdd    = $1 if ( $opts{p} =~ /(.+)/ );  my $PathAdd    = $1 if ( $opts{p} =~ /(.+)/ );
147  if ( $opts{s} !~ /^([\w\s\.\/\$-]+)$/ && $opts{s} ne "*" ) {  if ( $opts{s} !~ /^([\w\s\.\/\$-]+)$/ && $opts{s} ne "*" ) {
148      print(STDERR "$0: bad share name '$opts{s}'\n");      die "$0: bad share name '$opts{s}'\n";
     exit(1);  
149  }  }
150  our $ShareName = $opts{s};  our $ShareName = $opts{s};
151  our $view = BackupPC::View->new($bpc, $Host, \@Backups);  our $view = BackupPC::View->new($bpc, $Host, \@Backups);
# Line 164  my $sth_inc_size = $dbh->prepare(qq{ Line 162  my $sth_inc_size = $dbh->prepare(qq{
162                  inc_size = ?,                  inc_size = ?,
163                  parts = ?,                  parts = ?,
164                  inc_deleted = false                  inc_deleted = false
165          where id = ? });          where id = ?
166    });
167  my $sth_backup_parts = $dbh->prepare(qq{  my $sth_backup_parts = $dbh->prepare(qq{
168          insert into backup_parts (          insert into backup_parts (
169                  backup_id,                  backup_id,
# Line 207  die "problem with $tar_dir, check GzipTe Line 206  die "problem with $tar_dir, check GzipTe
206    
207  my $tar_file = BackupPC::SearchLib::getGzipName($Host, $ShareName, $Num) || die "can't getGzipName($Host, $ShareName, $Num)";  my $tar_file = BackupPC::SearchLib::getGzipName($Host, $ShareName, $Num) || die "can't getGzipName($Host, $ShareName, $Num)";
208    
209  my $tar_path = $tar_dir . '/' . $tar_file . '.tmp';  my $tar_path_final = $tar_dir . '/' . $tar_file;
210    my $tar_path = $tar_path_final . '.tmp';
211    
212  $tar_path =~ s#//#/#g;  $tar_path =~ s#//#/#g;
213    
214  my $sth = $dbh->prepare(qq{  my $sth = $dbh->prepare(qq{
# Line 224  $sth->finish; Line 225  $sth->finish;
225    
226  print STDERR "backup_id: $backup_id working dir: $tar_dir, max uncompressed size $max_file_size bytes, tar $tar_file\n" if ($opts{d});  print STDERR "backup_id: $backup_id working dir: $tar_dir, max uncompressed size $max_file_size bytes, tar $tar_file\n" if ($opts{d});
227    
228    if (-e $tar_path_final) {
229            if ($opts{f}) {
230                    rmtree $tar_path_final || die "can't remove $tar_path_final: $!";
231            } else {
232                    die "$tar_path_final allready exists\n";
233            }
234    }
235    
236  my $fh;  my $fh;
237  my $part = 0;  my $part = 0;
# Line 231  my $no_files = 0; Line 239  my $no_files = 0;
239  my $items_in_part = 0;  my $items_in_part = 0;
240    
241  sub new_tar_part {  sub new_tar_part {
242            my $arg = {@_};
243    
244          if ($fh) {          if ($fh) {
245                  return if ($current_tar_size == 0);                  return if ($current_tar_size == 0);
246    
247                  print STDERR "# closing part $part\n" if ($opts{d});                  print STDERR " $part";
248    
249                  # finish tar archive                  #
250                    # Finish with two null 512 byte headers,
251                    # and then round out a full block.
252                    #
253                  my $data = "\0" x ($tar_header_length * 2);                  my $data = "\0" x ($tar_header_length * 2);
254                  TarWrite($fh, \$data);                  TarWrite($fh, \$data);
255                  TarWrite($fh, undef);                  TarWrite($fh, undef);
# Line 259  sub new_tar_part { Line 272  sub new_tar_part {
272                          $items_in_part,                          $items_in_part,
273                  );                  );
274    
275                    $total_increment_size += $size;
276    
277                    if ($arg->{close}) {
278    
279                            sub move($$) {
280                                    my ($from,$to) = @_;
281                                    print STDERR "# rename $from -> $to\n" if ($opts{d});
282                                    rename $from, $to || die "can't move $from -> $to: $!\n";
283                            }
284    
285                            if ($part == 1) {
286                                    print STDERR " single" if ($opts{v});
287                                    move("${tar_path}/1.tar.gz", "${tar_path_final}.tar.gz");
288                                    move("${tar_path}/1.md5", "${tar_path_final}.md5");
289                                    rmtree $tar_path or die "can't remove temporary dir $tar_path: $!";
290                            } else {
291                                    print STDERR " [last]" if ($opts{v});
292                                    move("${tar_path}", "${tar_path_final}");
293                            }
294    
295                            $sth_inc_size->execute(
296                                    $total_increment_size,
297                                    $part,
298                                    $backup_id
299                            );
300    
301                            print STDERR ", $total_increment_size bytes\n" if ($opts{v});
302    
303                            return;
304                    }
305    
306          }          }
307    
308          $part++;          $part++;
# Line 266  sub new_tar_part { Line 310  sub new_tar_part {
310          # if this is first part, create directory          # if this is first part, create directory
311    
312          if ($part == 1) {          if ($part == 1) {
313                  if (-d $tar_path) {                  if (-e $tar_path) {
314                          print STDERR "# deleting existing $tar_path\n" if ($opts{d});                          print STDERR "# deleting existing $tar_path\n" if ($opts{d});
315                          rmtree($tar_path);                          rmtree($tar_path);
316                  }                  }
317                  mkdir($tar_path) || die "can't create directory $tar_path: $!";                  mkdir($tar_path) || die "can't create directory $tar_path: $!";
318    
319                    sub abort_cleanup {
320                            print STDERR "ABORTED: cleanup temp dir";
321                            rmtree($tar_path);
322                            $dbh->rollback;
323                            exit 1;
324                    }
325    
326                    $SIG{'INT'}  = \&abort_cleanup;
327                    $SIG{'QUIT'} = \&abort_cleanup;
328                    $SIG{'__DIE__'} = \&abort_cleanup;
329    
330          }          }
331    
332          my $file = $tar_path . '/' . $part;          my $file = $tar_path . '/' . $part;
# Line 299  new_tar_part(); Line 355  new_tar_part();
355  if (seedCache($Host, $ShareName, $Num)) {  if (seedCache($Host, $ShareName, $Num)) {
356          archiveWrite($fh, '/');          archiveWrite($fh, '/');
357          archiveWriteHardLinks($fh);          archiveWriteHardLinks($fh);
358            new_tar_part( close => 1 );
359  } else {  } else {
360          print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n" if ($opts{v});          print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n" if ($opts{v});
361          $no_files = 1;          # remove temporary files if there are no files
 }  
   
 #  
 # Finish with two null 512 byte headers, and then round out a full  
 # block.  
 #  
 my $data = "\0" x ($tar_header_length * 2);  
 TarWrite($fh, \$data);  
 TarWrite($fh, undef);  
   
 if (! close($fh)) {  
362          rmtree($tar_path);          rmtree($tar_path);
         die "can't close archive\n";  
 }  
   
 # remove temporary files if there are no files  
 if ($no_files) {  
         rmtree($tar_path);  
 } elsif ($part == 1) {  
         warn "FIXME: if there is only one part move to parent directory and rename";  
363  }  }
364    
365  #  #
# Line 336  if ( $ErrorCnt && !$FileCnt && !$DirCnt Line 374  if ( $ErrorCnt && !$FileCnt && !$DirCnt
374      # Got errors, with no files or directories; exit with non-zero      # Got errors, with no files or directories; exit with non-zero
375      # status      # status
376      #      #
377      cleanup();      die "got errors or no files\n";
     exit(1);  
378  }  }
379    
380  $sth_inc_size->finish;  $sth_inc_size->finish;
# Line 346  $sth_backup_parts->finish; Line 383  $sth_backup_parts->finish;
383  $dbh->commit || die "can't commit changes to database";  $dbh->commit || die "can't commit changes to database";
384  $dbh->disconnect();  $dbh->disconnect();
385    
386  exit(0);  exit;
387    
388  ###########################################################################  ###########################################################################
389  # Subroutines  # Subroutines
# Line 433  sub TarWrite Line 470  sub TarWrite
470      my $done = $WriteBufSz - length($WriteBuf);      my $done = $WriteBufSz - length($WriteBuf);
471      if ( syswrite($fh, $WriteBuf . substr($$dataRef, 0, $done))      if ( syswrite($fh, $WriteBuf . substr($$dataRef, 0, $done))
472                                  != $WriteBufSz ) {                                  != $WriteBufSz ) {
473          print(STDERR "Unable to write to output file ($!)\n");          die "Unable to write to output file ($!)\n";
         exit(1);  
474      }      }
475      while ( $done + $WriteBufSz <= length($$dataRef) ) {      while ( $done + $WriteBufSz <= length($$dataRef) ) {
476          if ( syswrite($fh, substr($$dataRef, $done, $WriteBufSz))          if ( syswrite($fh, substr($$dataRef, $done, $WriteBufSz))
477                              != $WriteBufSz ) {                              != $WriteBufSz ) {
478              print(STDERR "Unable to write to output file ($!)\n");              die "Unable to write to output file ($!)\n";
             exit(1);  
479          }          }
480          $done += $WriteBufSz;          $done += $WriteBufSz;
481      }      }
# Line 545  sub TarWriteFileInfo Line 580  sub TarWriteFileInfo
580  sub seedCache($$$) {  sub seedCache($$$) {
581          my ($host, $share, $dumpNo) = @_;          my ($host, $share, $dumpNo) = @_;
582    
583          print STDERR curr_time(), "getting files for $host:$share increment $dumpNo..." if ($opts{v});          print STDERR curr_time(), "$host:$share #$dumpNo" if ($opts{v});
584          my $sql = q{          my $sql = q{
585                  SELECT path,size                  SELECT path,size
586                  FROM files                  FROM files
# Line 557  sub seedCache($$$) { Line 592  sub seedCache($$$) {
592          my $sth = $dbh->prepare($sql);            my $sth = $dbh->prepare($sql);  
593          $sth->execute($host, $share, $dumpNo);          $sth->execute($host, $share, $dumpNo);
594          my $count = $sth->rows;          my $count = $sth->rows;
595          print STDERR " found $count items\n" if ($opts{v});          print STDERR " $count items, parts:" if ($opts{v});
596          while (my $row = $sth->fetchrow_arrayref) {          while (my $row = $sth->fetchrow_arrayref) {
597  #print STDERR "+ ", $row->[0],"\n";  #print STDERR "+ ", $row->[0],"\n";
598                  $in_backup_increment->{ $row->[0] } = $row->[1];                  $in_backup_increment->{ $row->[0] } = $row->[1];
# Line 605  sub TarWriteFile Line 640  sub TarWriteFile
640      # is this file too large to fit into MaxArchiveFileSize?      # is this file too large to fit into MaxArchiveFileSize?
641    
642      if ( ($current_tar_size + tar_overhead($tarPath) + $size) > $max_file_size ) {      if ( ($current_tar_size + tar_overhead($tarPath) + $size) > $max_file_size ) {
643          print STDERR "# tar file $current_tar_size + $tar_header_length + $size > $max_file_size, splitting\n" if ($opts{d});          print STDERR "# tar file $current_tar_size + $tar_header_length + $size > $max_file_size, splitting\n" if ($opts{d});
644          new_tar_part();          new_tar_part();
645      }      }
646    
Legend:
Removed from v.235  
changed lines
  Added in v.254
  ViewVC Help
Powered by ViewVC 1.1.26