/[BackupPC]/trunk/bin/BackupPC_tarIncCreate
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/bin/BackupPC_tarIncCreate

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 156 by dpavlin, Mon Oct 10 13:04:49 2005 UTC revision 234 by dpavlin, Tue Nov 8 20:24:45 2005 UTC
# Line 1  Line 1 
1  #!/usr/bin/perl  #!/usr/bin/perl -w
2  #============================================================= -*-perl-*-  #============================================================= -*-perl-*-
3  #  #
4  # BackupPC_tarIncCreate: create a tar archive of an existing incremental dump  # BackupPC_tarIncCreate: create a tar archive of an existing incremental dump
# Line 72  use BackupPC::View; Line 72  use BackupPC::View;
72  use BackupPC::SearchLib;  use BackupPC::SearchLib;
73  use Time::HiRes qw/time/;  use Time::HiRes qw/time/;
74  use POSIX qw/strftime/;  use POSIX qw/strftime/;
75    use File::Which;
76    use File::Path;
77  use Data::Dumper;       ### FIXME  use Data::Dumper;       ### FIXME
78    
79  die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) );  die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) );
80  my $TopDir = $bpc->TopDir();  my $TopDir = $bpc->TopDir();
81  my $BinDir = $bpc->BinDir();  my $BinDir = $bpc->BinDir();
82  my %Conf   = $bpc->Conf();  my %Conf   = $bpc->Conf();
83    %BackupPC::SearchLib::Conf = %Conf;
84  my %opts;  my %opts;
85  my $in_backup_increment;  my $in_backup_increment;
86    
87    
88  if ( !getopts("th:n:p:r:s:b:w:v", \%opts) ) {  if ( !getopts("th:n:p:r:s:b:w:vd", \%opts) ) {
89      print STDERR <<EOF;      print STDERR <<EOF;
90  usage: $0 [options]  usage: $0 [options]
91    Required options:    Required options:
# Line 99  usage: $0 [options] Line 102  usage: $0 [options]
102       -b BLOCKS       BLOCKS x 512 bytes per record (default 20; same as tar)       -b BLOCKS       BLOCKS x 512 bytes per record (default 20; same as tar)
103       -w writeBufSz   write buffer size (default 1048576 = 1MB)       -w writeBufSz   write buffer size (default 1048576 = 1MB)
104       -v              verbose output       -v              verbose output
105         -d              debug output
106  EOF  EOF
107      exit(1);      exit(1);
108  }  }
# Line 115  if ( $opts{n} !~ /^(-?\d+)$/ ) { Line 119  if ( $opts{n} !~ /^(-?\d+)$/ ) {
119  }  }
120  my $Num = $opts{n};  my $Num = $opts{n};
121    
122    my $bin;
123    foreach my $c (qw/gzip md5sum tee/) {
124            $bin->{$c} = which($c) || die "$0 needs $c, install it\n";
125    }
126    
127  my @Backups = $bpc->BackupInfoRead($Host);  my @Backups = $bpc->BackupInfoRead($Host);
128  my $FileCnt = 0;  my $FileCnt = 0;
129  my $ByteCnt = 0;  my $ByteCnt = 0;
130  my $DirCnt = 0;  my $DirCnt = 0;
131  my $SpecialCnt = 0;  my $SpecialCnt = 0;
132  my $ErrorCnt = 0;  my $ErrorCnt = 0;
133    my $current_tar_size = 0;
134    
135  my $i;  my $i;
136  $Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 );  $Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 );
# Line 163  my(%HardLinkExtraFiles, @HardLinks); Line 173  my(%HardLinkExtraFiles, @HardLinks);
173  #  #
174  # Write out all the requested files/directories  # Write out all the requested files/directories
175  #  #
176  binmode(STDOUT);  
177  my $fh = *STDOUT;  my $max_file_size = $Conf{'MaxArchiveFileSize'} || die "problem with MaxArchiveFileSize parametar";
178    $max_file_size *= 1024;
179    
180    my $tar_dir = $Conf{InstallDir}.'/'.$Conf{GzipTempDir};
181    die "problem with $tar_dir, check GzipTempDir in configuration\n" unless (-d $tar_dir && -w $tar_dir);
182    
183    my $tar_file = BackupPC::SearchLib::getGzipName($Host, $ShareName, $Num) || die "can't getGzipName($Host, $ShareName, $Num)";
184    
185    my $tar_path = $tar_dir . '/' . $tar_file . '.tmp';
186    $tar_path =~ s#//#/#g;
187    
188    print STDERR "working dir: $tar_dir, max uncompressed size $max_file_size bytes, tar $tar_file\n" if ($opts{d});
189    
190    my $fh;
191    my $part = 0;
192    my $no_files = 0;
193    
194    sub new_tar_part {
195            if ($fh) {
196                    return if ($current_tar_size == 0);
197    
198                    print STDERR "# closing part $part\n" if ($opts{d});
199    
200                    # finish tar archive
201                    my $data = "\0" x ($tar_header_length * 2);
202                    TarWrite($fh, \$data);
203                    TarWrite($fh, undef);
204    
205                    close($fh) || die "can't close archive part $part: $!";
206            }
207    
208            $part++;
209    
210            # if this is first part, create directory
211    
212            if ($part == 1) {
213                    if (-d $tar_path) {
214                            print STDERR "# deleting existing $tar_path\n" if ($opts{d});
215                            rmtree($tar_path);
216                    }
217                    mkdir($tar_path) || die "can't create directory $tar_path: $!";
218            }
219    
220            my $file = $tar_path . '/' . $part;
221    
222            #
223            # create comprex pipe which will pass output through gzip
224            # for compression, create file on disk using tee
225            # and pipe same output to md5sum to create checksum
226            #
227    
228            my $cmd = '| ' . $bin->{'gzip'}   . ' ' . $Conf{GzipLevel} .      ' ' .
229                      '| ' . $bin->{'tee'}    . ' ' . $file . '.tar.gz' . ' ' .
230                      '| ' . $bin->{'md5sum'} . ' - > ' . $file . '.md5';
231    
232            print STDERR "## $cmd\n" if ($opts{d});
233    
234            open($fh, $cmd) or die "can't open $cmd: $!";
235            binmode($fh);
236            $current_tar_size = 0;
237    }
238    
239    new_tar_part();
240    
241  if (seedCache($Host, $ShareName, $Num)) {  if (seedCache($Host, $ShareName, $Num)) {
242          archiveWrite($fh, '/');          archiveWrite($fh, '/');
243          archiveWriteHardLinks($fh);          archiveWriteHardLinks($fh);
244  } else {  } else {
245          print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n";          print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n" if ($opts{v});
246            $no_files = 1;
247  }  }
248    
249  #  #
# Line 181  my $data = "\0" x ($tar_header_length * Line 254  my $data = "\0" x ($tar_header_length *
254  TarWrite($fh, \$data);  TarWrite($fh, \$data);
255  TarWrite($fh, undef);  TarWrite($fh, undef);
256    
257    if (! close($fh)) {
258            rmtree($tar_path);
259            die "can't close archive\n";
260    }
261    
262    # remove temporary files if there are no files
263    if ($no_files) {
264            rmtree($tar_path);
265    } elsif ($part == 1) {
266            warn "FIXME: if there is only one part move to parent directory and rename";
267    }
268    
269  #  #
270  # print out totals if requested  # print out totals if requested
271  #  #
# Line 193  if ( $ErrorCnt && !$FileCnt && !$DirCnt Line 278  if ( $ErrorCnt && !$FileCnt && !$DirCnt
278      # Got errors, with no files or directories; exit with non-zero      # Got errors, with no files or directories; exit with non-zero
279      # status      # status
280      #      #
281        cleanup();
282      exit(1);      exit(1);
283  }  }
284    
285  exit(0);  exit(0);
286    
287  ###########################################################################  ###########################################################################
# Line 260  sub TarWrite Line 347  sub TarWrite
347  {  {
348      my($fh, $dataRef) = @_;      my($fh, $dataRef) = @_;
349    
350    
351      if ( !defined($dataRef) ) {      if ( !defined($dataRef) ) {
352          #          #
353          # do flush by padding to a full $WriteBufSz          # do flush by padding to a full $WriteBufSz
# Line 267  sub TarWrite Line 355  sub TarWrite
355          my $data = "\0" x ($WriteBufSz - length($WriteBuf));          my $data = "\0" x ($WriteBufSz - length($WriteBuf));
356          $dataRef = \$data;          $dataRef = \$data;
357      }      }
358    
359        # poor man's tell :-)
360        $current_tar_size += length($$dataRef);
361    
362      if ( length($WriteBuf) + length($$dataRef) < $WriteBufSz ) {      if ( length($WriteBuf) + length($$dataRef) < $WriteBufSz ) {
363          #          #
364          # just buffer and return          # just buffer and return
# Line 394  sub seedCache($$$) { Line 486  sub seedCache($$$) {
486                    
487          print STDERR curr_time(), "getting files for $host:$share increment $dumpNo..." if ($opts{v});          print STDERR curr_time(), "getting files for $host:$share increment $dumpNo..." if ($opts{v});
488          my $sql = q{          my $sql = q{
489                  SELECT path                  SELECT path,size
490                  FROM files                  FROM files
491                          JOIN shares on shares.id = shareid                          JOIN shares on shares.id = shareid
492                          JOIN hosts on hosts.id = shares.hostid                          JOIN hosts on hosts.id = shares.hostid
# Line 408  sub seedCache($$$) { Line 500  sub seedCache($$$) {
500          print STDERR " found $count items\n" if ($opts{v});          print STDERR " found $count items\n" if ($opts{v});
501          while (my $row = $sth->fetchrow_arrayref) {          while (my $row = $sth->fetchrow_arrayref) {
502  #print STDERR "+ ", $row->[0],"\n";  #print STDERR "+ ", $row->[0],"\n";
503                  $in_backup_increment->{ $row->[0] }++;                  $in_backup_increment->{ $row->[0] } = $row->[1];
504          }          }
505                    
506          $sth->finish();          $sth->finish();
# Line 417  sub seedCache($$$) { Line 509  sub seedCache($$$) {
509          return $count;          return $count;
510  }  }
511    
512    #
513    # calculate overhad for one file in tar
514    #
515    sub tar_overhead($) {
516            my $name = shift || '';
517    
518            # header, padding of file and two null blocks at end
519            my $len = 4 * $tar_header_length;
520    
521            # if filename is longer than 99 chars subtract blocks for
522            # long filename
523            if ( length($name) > 99 ) {
524                    $len += int( ( length($name) + $tar_header_length ) / $tar_header_length ) * $tar_header_length;
525            }
526    
527            return $len;
528    }
529    
530  my $Attr;  my $Attr;
531  my $AttrDir;  my $AttrDir;
532    
# Line 429  sub TarWriteFile Line 539  sub TarWriteFile
539    
540      $tarPath =~ s{//+}{/}g;      $tarPath =~ s{//+}{/}g;
541    
542  #print STDERR "? $tarPath\n";      #print STDERR "? $tarPath\n" if ($opts{d});
543      return unless ($in_backup_increment->{$tarPath});      my $size = $in_backup_increment->{$tarPath};
544  #print STDERR "A $tarPath\n";      return unless (defined($size));
545    
546        # is this file too large to fit into MaxArchiveFileSize?
547    
548        if ( ($current_tar_size + tar_overhead($tarPath) + $size) > $max_file_size ) {
549            print STDERR "# tar file $current_tar_size + $tar_header_length + $size > $max_file_size, splitting\n" if ($opts{d});
550            new_tar_part();
551        }
552    
553        print STDERR "A $tarPath [$size] tell: $current_tar_size\n" if ($opts{d});
554    
555      if ( defined($PathRemove)      if ( defined($PathRemove)
556              && substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) {              && substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) {
# Line 445  sub TarWriteFile Line 564  sub TarWriteFile
564          #          #
565          # Directory: just write the header          # Directory: just write the header
566          #          #
           
                   
567          $hdr->{name} .= "/" if ( $hdr->{name} !~ m{/$} );          $hdr->{name} .= "/" if ( $hdr->{name} !~ m{/$} );
568          TarWriteFileInfo($fh, $hdr);          TarWriteFileInfo($fh, $hdr);
569          $DirCnt++;          $DirCnt++;
# Line 460  sub TarWriteFile Line 577  sub TarWriteFile
577              $ErrorCnt++;              $ErrorCnt++;
578              return;              return;
579          }          }
580          TarWriteFileInfo($fh, $hdr);          # do we need to split file?
581          my($data, $size);          if ($hdr->{size} < $max_file_size) {
582          while ( $f->read(\$data, $BufSize) > 0 ) {                  TarWriteFileInfo($fh, $hdr);
583              TarWrite($fh, \$data);                  my($data, $size);
584              $size += length($data);                  while ( $f->read(\$data, $BufSize) > 0 ) {
585          }                      TarWrite($fh, \$data);
586          $f->close;                      $size += length($data);
587          TarWritePad($fh, $size);                  }
588                    $f->close;
589                    TarWritePad($fh, $size);
590                  $FileCnt++;                  $FileCnt++;
591                  $ByteCnt += $size;                  $ByteCnt += $size;
592            } else {
593                    my $full_size = $hdr->{size};
594                    my $orig_name = $hdr->{name};
595                    my $max_part_size = $max_file_size - tar_overhead($hdr->{name});
596    
597                    my $parts = int(($full_size + $max_part_size - 1) / $max_part_size);
598                    print STDERR "# splitting $orig_name [$full_size bytes] into $parts parts\n" if ($opts{d});
599                    foreach my $subpart ( 1 .. $parts ) {
600                            new_tar_part();
601                            if ($subpart < $parts) {
602                                    $hdr->{size} = $max_part_size;
603                            } else {
604                                    $hdr->{size} = $full_size % $max_part_size;
605                            }
606                            $hdr->{name} = $orig_name . '/' . $subpart;
607                            print STDERR "## creating part $subpart ",$hdr->{name}, " [", $hdr->{size}," bytes]\n";
608    
609                            TarWriteFileInfo($fh, $hdr);
610                            my($data, $size);
611    if (0) {
612                            for ( 1 .. int($hdr->{size} / $BufSize) ) {
613                                    my $r_size = $f->read(\$data, $BufSize);
614                                    die "expected $BufSize bytes read, got $r_size bytes!" if ($r_size != $BufSize);
615                                    TarWrite($fh, \$data);
616                                    $size += length($data);
617                            }
618    }
619                            my $size_left = $hdr->{size} % $BufSize;
620                            my $r_size = $f->read(\$data, $size_left);
621                            die "expected $size_left bytes last read, got $r_size bytes!" if ($r_size != $size_left);
622    
623                            TarWrite($fh, \$data);
624                            $size += length($data);
625                            TarWritePad($fh, $size);
626                    }
627                    $f->close;
628                    $FileCnt++;
629                    $ByteCnt += $full_size;
630                    new_tar_part();
631            }
632      } elsif ( $hdr->{type} == BPC_FTYPE_HARDLINK ) {      } elsif ( $hdr->{type} == BPC_FTYPE_HARDLINK ) {
633          #          #
634          # Hardlink file: either write a hardlink or the complete file          # Hardlink file: either write a hardlink or the complete file
635                  # depending upon whether the linked-to file will be written          # depending upon whether the linked-to file will be written
636                  # to the archive.          # to the archive.
637          #          #
638                  # Start by reading the contents of the link.          # Start by reading the contents of the link.
639                  #          #
640          my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress});          my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress});
641          if ( !defined($f) ) {          if ( !defined($f) ) {
642              print(STDERR "Unable to open file $hdr->{fullPath}\n");              print(STDERR "Unable to open file $hdr->{fullPath}\n");
# Line 488  sub TarWriteFile Line 647  sub TarWriteFile
647          while ( $f->read(\$data, $BufSize) > 0 ) {          while ( $f->read(\$data, $BufSize) > 0 ) {
648              $hdr->{linkname} .= $data;              $hdr->{linkname} .= $data;
649          }          }
650                  $f->close;          $f->close;
651                  my $done = 0;          my $done = 0;
652                  my $name = $hdr->{linkname};          my $name = $hdr->{linkname};
653                  $name =~ s{^\./}{/};          $name =~ s{^\./}{/};
654                  if ( $HardLinkExtraFiles{$name} ) {          if ( $HardLinkExtraFiles{$name} ) {
655                      #              #
656                      # Target file will be or was written, so just remember              # Target file will be or was written, so just remember
657                      # the hardlink so we can dump it later.              # the hardlink so we can dump it later.
658                      #              #
659                      push(@HardLinks, $hdr);              push(@HardLinks, $hdr);
660                      $SpecialCnt++;              $SpecialCnt++;
661                  } else {          } else {
662                      #              #
663                      # Have to dump the original file.  Just call the top-level              # Have to dump the original file.  Just call the top-level
664                      # routine, so that we save the hassle of dealing with              # routine, so that we save the hassle of dealing with
665                      # mangling, merging and attributes.              # mangling, merging and attributes.
666                      #              #
667                      $HardLinkExtraFiles{$hdr->{linkname}} = 1;              $HardLinkExtraFiles{$hdr->{linkname}} = 1;
668                      archiveWrite($fh, $hdr->{linkname}, $hdr->{name});              archiveWrite($fh, $hdr->{linkname}, $hdr->{name});
669                  }          }
670      } elsif ( $hdr->{type} == BPC_FTYPE_SYMLINK ) {      } elsif ( $hdr->{type} == BPC_FTYPE_SYMLINK ) {
671          #          #
672          # Symbolic link: read the symbolic link contents into the header          # Symbolic link: read the symbolic link contents into the header
Legend:
Removed from v.156  
changed lines
  Added in v.234
  ViewVC Help
Powered by ViewVC 1.1.26