1 |
#!/usr/bin/perl |
#!/usr/bin/perl -w |
2 |
#============================================================= -*-perl-*- |
#============================================================= -*-perl-*- |
3 |
# |
# |
4 |
# BackupPC_tarIncCreate: create a tar archive of an existing incremental dump |
# BackupPC_tarIncCreate: create a tar archive of an existing incremental dump |
72 |
use BackupPC::SearchLib; |
use BackupPC::SearchLib; |
73 |
use Time::HiRes qw/time/; |
use Time::HiRes qw/time/; |
74 |
use POSIX qw/strftime/; |
use POSIX qw/strftime/; |
75 |
|
use File::Which; |
76 |
|
use File::Path; |
77 |
use Data::Dumper; ### FIXME |
use Data::Dumper; ### FIXME |
78 |
|
|
79 |
die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); |
die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); |
80 |
my $TopDir = $bpc->TopDir(); |
my $TopDir = $bpc->TopDir(); |
81 |
my $BinDir = $bpc->BinDir(); |
my $BinDir = $bpc->BinDir(); |
82 |
my %Conf = $bpc->Conf(); |
my %Conf = $bpc->Conf(); |
83 |
|
%BackupPC::SearchLib::Conf = %Conf; |
84 |
my %opts; |
my %opts; |
85 |
my $in_backup_increment; |
my $in_backup_increment; |
86 |
|
|
87 |
|
|
88 |
if ( !getopts("th:n:p:r:s:b:w:v", \%opts) ) { |
if ( !getopts("th:n:p:r:s:b:w:vd", \%opts) ) { |
89 |
print STDERR <<EOF; |
print STDERR <<EOF; |
90 |
usage: $0 [options] |
usage: $0 [options] |
91 |
Required options: |
Required options: |
102 |
-b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar) |
-b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar) |
103 |
-w writeBufSz write buffer size (default 1048576 = 1MB) |
-w writeBufSz write buffer size (default 1048576 = 1MB) |
104 |
-v verbose output |
-v verbose output |
105 |
|
-d debug output |
106 |
EOF |
EOF |
107 |
exit(1); |
exit(1); |
108 |
} |
} |
119 |
} |
} |
120 |
my $Num = $opts{n}; |
my $Num = $opts{n}; |
121 |
|
|
122 |
|
my $bin; |
123 |
|
foreach my $c (qw/gzip md5sum tee/) { |
124 |
|
$bin->{$c} = which($c) || die "$0 needs $c, install it\n"; |
125 |
|
} |
126 |
|
|
127 |
my @Backups = $bpc->BackupInfoRead($Host); |
my @Backups = $bpc->BackupInfoRead($Host); |
128 |
my $FileCnt = 0; |
my $FileCnt = 0; |
129 |
my $ByteCnt = 0; |
my $ByteCnt = 0; |
130 |
my $DirCnt = 0; |
my $DirCnt = 0; |
131 |
my $SpecialCnt = 0; |
my $SpecialCnt = 0; |
132 |
my $ErrorCnt = 0; |
my $ErrorCnt = 0; |
133 |
|
my $current_tar_size = 0; |
134 |
|
|
135 |
my $i; |
my $i; |
136 |
$Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 ); |
$Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 ); |
173 |
# |
# |
174 |
# Write out all the requested files/directories |
# Write out all the requested files/directories |
175 |
# |
# |
176 |
binmode(STDOUT); |
|
177 |
my $fh = *STDOUT; |
my $max_file_size = $Conf{'MaxArchiveFileSize'} || die "problem with MaxArchiveFileSize parametar"; |
178 |
|
$max_file_size *= 1024; |
179 |
|
|
180 |
|
my $tar_dir = $Conf{InstallDir}.'/'.$Conf{GzipTempDir}; |
181 |
|
die "problem with $tar_dir, check GzipTempDir in configuration\n" unless (-d $tar_dir && -w $tar_dir); |
182 |
|
|
183 |
|
my $tar_file = BackupPC::SearchLib::getGzipName($Host, $ShareName, $Num) || die "can't getGzipName($Host, $ShareName, $Num)"; |
184 |
|
|
185 |
|
my $tar_path = $tar_dir . '/' . $tar_file . '.tmp'; |
186 |
|
$tar_path =~ s#//#/#g; |
187 |
|
|
188 |
|
print STDERR "working dir: $tar_dir, max uncompressed size $max_file_size bytes, tar $tar_file\n" if ($opts{d}); |
189 |
|
|
190 |
|
my $fh; |
191 |
|
my $part = 0; |
192 |
|
my $no_files = 0; |
193 |
|
|
194 |
|
sub new_tar_part { |
195 |
|
if ($fh) { |
196 |
|
return if ($current_tar_size == 0); |
197 |
|
|
198 |
|
print STDERR "# closing part $part\n" if ($opts{d}); |
199 |
|
|
200 |
|
# finish tar archive |
201 |
|
my $data = "\0" x ($tar_header_length * 2); |
202 |
|
TarWrite($fh, \$data); |
203 |
|
TarWrite($fh, undef); |
204 |
|
|
205 |
|
close($fh) || die "can't close archive part $part: $!"; |
206 |
|
} |
207 |
|
|
208 |
|
$part++; |
209 |
|
|
210 |
|
# if this is first part, create directory |
211 |
|
|
212 |
|
if ($part == 1) { |
213 |
|
if (-d $tar_path) { |
214 |
|
print STDERR "# deleting existing $tar_path\n" if ($opts{d}); |
215 |
|
rmtree($tar_path); |
216 |
|
} |
217 |
|
mkdir($tar_path) || die "can't create directory $tar_path: $!"; |
218 |
|
} |
219 |
|
|
220 |
|
my $file = $tar_path . '/' . $part; |
221 |
|
|
222 |
|
# |
223 |
|
# create comprex pipe which will pass output through gzip |
224 |
|
# for compression, create file on disk using tee |
225 |
|
# and pipe same output to md5sum to create checksum |
226 |
|
# |
227 |
|
|
228 |
|
my $cmd = '| ' . $bin->{'gzip'} . ' ' . $Conf{GzipLevel} . ' ' . |
229 |
|
'| ' . $bin->{'tee'} . ' ' . $file . '.tar.gz' . ' ' . |
230 |
|
'| ' . $bin->{'md5sum'} . ' - > ' . $file . '.md5'; |
231 |
|
|
232 |
|
print STDERR "## $cmd\n" if ($opts{d}); |
233 |
|
|
234 |
|
open($fh, $cmd) or die "can't open $cmd: $!"; |
235 |
|
binmode($fh); |
236 |
|
$current_tar_size = 0; |
237 |
|
} |
238 |
|
|
239 |
|
new_tar_part(); |
240 |
|
|
241 |
if (seedCache($Host, $ShareName, $Num)) { |
if (seedCache($Host, $ShareName, $Num)) { |
242 |
archiveWrite($fh, '/'); |
archiveWrite($fh, '/'); |
243 |
archiveWriteHardLinks($fh); |
archiveWriteHardLinks($fh); |
244 |
} else { |
} else { |
245 |
print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n" if ($opts{v}); |
print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n" if ($opts{v}); |
246 |
|
$no_files = 1; |
247 |
} |
} |
248 |
|
|
249 |
# |
# |
254 |
TarWrite($fh, \$data); |
TarWrite($fh, \$data); |
255 |
TarWrite($fh, undef); |
TarWrite($fh, undef); |
256 |
|
|
257 |
|
if (! close($fh)) { |
258 |
|
rmtree($tar_path); |
259 |
|
die "can't close archive\n"; |
260 |
|
} |
261 |
|
|
262 |
|
# remove temporary files if there are no files |
263 |
|
if ($no_files) { |
264 |
|
rmtree($tar_path); |
265 |
|
} elsif ($part == 1) { |
266 |
|
warn "FIXME: if there is only one part move to parent directory and rename"; |
267 |
|
} |
268 |
|
|
269 |
# |
# |
270 |
# print out totals if requested |
# print out totals if requested |
271 |
# |
# |
278 |
# Got errors, with no files or directories; exit with non-zero |
# Got errors, with no files or directories; exit with non-zero |
279 |
# status |
# status |
280 |
# |
# |
281 |
|
cleanup(); |
282 |
exit(1); |
exit(1); |
283 |
} |
} |
284 |
|
|
285 |
exit(0); |
exit(0); |
286 |
|
|
287 |
########################################################################### |
########################################################################### |
347 |
{ |
{ |
348 |
my($fh, $dataRef) = @_; |
my($fh, $dataRef) = @_; |
349 |
|
|
350 |
|
|
351 |
if ( !defined($dataRef) ) { |
if ( !defined($dataRef) ) { |
352 |
# |
# |
353 |
# do flush by padding to a full $WriteBufSz |
# do flush by padding to a full $WriteBufSz |
355 |
my $data = "\0" x ($WriteBufSz - length($WriteBuf)); |
my $data = "\0" x ($WriteBufSz - length($WriteBuf)); |
356 |
$dataRef = \$data; |
$dataRef = \$data; |
357 |
} |
} |
358 |
|
|
359 |
|
# poor man's tell :-) |
360 |
|
$current_tar_size += length($$dataRef); |
361 |
|
|
362 |
if ( length($WriteBuf) + length($$dataRef) < $WriteBufSz ) { |
if ( length($WriteBuf) + length($$dataRef) < $WriteBufSz ) { |
363 |
# |
# |
364 |
# just buffer and return |
# just buffer and return |
486 |
|
|
487 |
print STDERR curr_time(), "getting files for $host:$share increment $dumpNo..." if ($opts{v}); |
print STDERR curr_time(), "getting files for $host:$share increment $dumpNo..." if ($opts{v}); |
488 |
my $sql = q{ |
my $sql = q{ |
489 |
SELECT path |
SELECT path,size |
490 |
FROM files |
FROM files |
491 |
JOIN shares on shares.id = shareid |
JOIN shares on shares.id = shareid |
492 |
JOIN hosts on hosts.id = shares.hostid |
JOIN hosts on hosts.id = shares.hostid |
500 |
print STDERR " found $count items\n" if ($opts{v}); |
print STDERR " found $count items\n" if ($opts{v}); |
501 |
while (my $row = $sth->fetchrow_arrayref) { |
while (my $row = $sth->fetchrow_arrayref) { |
502 |
#print STDERR "+ ", $row->[0],"\n"; |
#print STDERR "+ ", $row->[0],"\n"; |
503 |
$in_backup_increment->{ $row->[0] }++; |
$in_backup_increment->{ $row->[0] } = $row->[1]; |
504 |
} |
} |
505 |
|
|
506 |
$sth->finish(); |
$sth->finish(); |
509 |
return $count; |
return $count; |
510 |
} |
} |
511 |
|
|
512 |
|
# |
513 |
|
# calculate overhad for one file in tar |
514 |
|
# |
515 |
|
sub tar_overhead($) { |
516 |
|
my $name = shift || ''; |
517 |
|
|
518 |
|
# header, padding of file and two null blocks at end |
519 |
|
my $len = 4 * $tar_header_length; |
520 |
|
|
521 |
|
# if filename is longer than 99 chars subtract blocks for |
522 |
|
# long filename |
523 |
|
if ( length($name) > 99 ) { |
524 |
|
$len += int( ( length($name) + $tar_header_length ) / $tar_header_length ) * $tar_header_length; |
525 |
|
} |
526 |
|
|
527 |
|
return $len; |
528 |
|
} |
529 |
|
|
530 |
my $Attr; |
my $Attr; |
531 |
my $AttrDir; |
my $AttrDir; |
532 |
|
|
539 |
|
|
540 |
$tarPath =~ s{//+}{/}g; |
$tarPath =~ s{//+}{/}g; |
541 |
|
|
542 |
#print STDERR "? $tarPath\n"; |
#print STDERR "? $tarPath\n" if ($opts{d}); |
543 |
return unless ($in_backup_increment->{$tarPath}); |
my $size = $in_backup_increment->{$tarPath}; |
544 |
#print STDERR "A $tarPath\n"; |
return unless (defined($size)); |
545 |
|
|
546 |
|
# is this file too large to fit into MaxArchiveFileSize? |
547 |
|
|
548 |
|
if ( ($current_tar_size + tar_overhead($tarPath) + $size) > $max_file_size ) { |
549 |
|
print STDERR "# tar file $current_tar_size + $tar_header_length + $size > $max_file_size, splitting\n" if ($opts{d}); |
550 |
|
new_tar_part(); |
551 |
|
} |
552 |
|
|
553 |
|
print STDERR "A $tarPath [$size] tell: $current_tar_size\n" if ($opts{d}); |
554 |
|
|
555 |
if ( defined($PathRemove) |
if ( defined($PathRemove) |
556 |
&& substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) { |
&& substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) { |
564 |
# |
# |
565 |
# Directory: just write the header |
# Directory: just write the header |
566 |
# |
# |
|
|
|
|
|
|
567 |
$hdr->{name} .= "/" if ( $hdr->{name} !~ m{/$} ); |
$hdr->{name} .= "/" if ( $hdr->{name} !~ m{/$} ); |
568 |
TarWriteFileInfo($fh, $hdr); |
TarWriteFileInfo($fh, $hdr); |
569 |
$DirCnt++; |
$DirCnt++; |
577 |
$ErrorCnt++; |
$ErrorCnt++; |
578 |
return; |
return; |
579 |
} |
} |
580 |
TarWriteFileInfo($fh, $hdr); |
# do we need to split file? |
581 |
my($data, $size); |
if ($hdr->{size} < $max_file_size) { |
582 |
while ( $f->read(\$data, $BufSize) > 0 ) { |
TarWriteFileInfo($fh, $hdr); |
583 |
TarWrite($fh, \$data); |
my($data, $size); |
584 |
$size += length($data); |
while ( $f->read(\$data, $BufSize) > 0 ) { |
585 |
} |
TarWrite($fh, \$data); |
586 |
$f->close; |
$size += length($data); |
587 |
TarWritePad($fh, $size); |
} |
588 |
|
$f->close; |
589 |
|
TarWritePad($fh, $size); |
590 |
$FileCnt++; |
$FileCnt++; |
591 |
$ByteCnt += $size; |
$ByteCnt += $size; |
592 |
|
} else { |
593 |
|
my $full_size = $hdr->{size}; |
594 |
|
my $orig_name = $hdr->{name}; |
595 |
|
my $max_part_size = $max_file_size - tar_overhead($hdr->{name}); |
596 |
|
|
597 |
|
my $parts = int(($full_size + $max_part_size - 1) / $max_part_size); |
598 |
|
print STDERR "# splitting $orig_name [$full_size bytes] into $parts parts\n" if ($opts{d}); |
599 |
|
foreach my $subpart ( 1 .. $parts ) { |
600 |
|
new_tar_part(); |
601 |
|
if ($subpart < $parts) { |
602 |
|
$hdr->{size} = $max_part_size; |
603 |
|
} else { |
604 |
|
$hdr->{size} = $full_size % $max_part_size; |
605 |
|
} |
606 |
|
$hdr->{name} = $orig_name . '/' . $subpart; |
607 |
|
print STDERR "## creating part $subpart ",$hdr->{name}, " [", $hdr->{size}," bytes]\n"; |
608 |
|
|
609 |
|
TarWriteFileInfo($fh, $hdr); |
610 |
|
my($data, $size); |
611 |
|
if (0) { |
612 |
|
for ( 1 .. int($hdr->{size} / $BufSize) ) { |
613 |
|
my $r_size = $f->read(\$data, $BufSize); |
614 |
|
die "expected $BufSize bytes read, got $r_size bytes!" if ($r_size != $BufSize); |
615 |
|
TarWrite($fh, \$data); |
616 |
|
$size += length($data); |
617 |
|
} |
618 |
|
} |
619 |
|
my $size_left = $hdr->{size} % $BufSize; |
620 |
|
my $r_size = $f->read(\$data, $size_left); |
621 |
|
die "expected $size_left bytes last read, got $r_size bytes!" if ($r_size != $size_left); |
622 |
|
|
623 |
|
TarWrite($fh, \$data); |
624 |
|
$size += length($data); |
625 |
|
TarWritePad($fh, $size); |
626 |
|
} |
627 |
|
$f->close; |
628 |
|
$FileCnt++; |
629 |
|
$ByteCnt += $full_size; |
630 |
|
new_tar_part(); |
631 |
|
} |
632 |
} elsif ( $hdr->{type} == BPC_FTYPE_HARDLINK ) { |
} elsif ( $hdr->{type} == BPC_FTYPE_HARDLINK ) { |
633 |
# |
# |
634 |
# Hardlink file: either write a hardlink or the complete file |
# Hardlink file: either write a hardlink or the complete file |
635 |
# depending upon whether the linked-to file will be written |
# depending upon whether the linked-to file will be written |
636 |
# to the archive. |
# to the archive. |
637 |
# |
# |
638 |
# Start by reading the contents of the link. |
# Start by reading the contents of the link. |
639 |
# |
# |
640 |
my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress}); |
my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress}); |
641 |
if ( !defined($f) ) { |
if ( !defined($f) ) { |
642 |
print(STDERR "Unable to open file $hdr->{fullPath}\n"); |
print(STDERR "Unable to open file $hdr->{fullPath}\n"); |
647 |
while ( $f->read(\$data, $BufSize) > 0 ) { |
while ( $f->read(\$data, $BufSize) > 0 ) { |
648 |
$hdr->{linkname} .= $data; |
$hdr->{linkname} .= $data; |
649 |
} |
} |
650 |
$f->close; |
$f->close; |
651 |
my $done = 0; |
my $done = 0; |
652 |
my $name = $hdr->{linkname}; |
my $name = $hdr->{linkname}; |
653 |
$name =~ s{^\./}{/}; |
$name =~ s{^\./}{/}; |
654 |
if ( $HardLinkExtraFiles{$name} ) { |
if ( $HardLinkExtraFiles{$name} ) { |
655 |
# |
# |
656 |
# Target file will be or was written, so just remember |
# Target file will be or was written, so just remember |
657 |
# the hardlink so we can dump it later. |
# the hardlink so we can dump it later. |
658 |
# |
# |
659 |
push(@HardLinks, $hdr); |
push(@HardLinks, $hdr); |
660 |
$SpecialCnt++; |
$SpecialCnt++; |
661 |
} else { |
} else { |
662 |
# |
# |
663 |
# Have to dump the original file. Just call the top-level |
# Have to dump the original file. Just call the top-level |
664 |
# routine, so that we save the hassle of dealing with |
# routine, so that we save the hassle of dealing with |
665 |
# mangling, merging and attributes. |
# mangling, merging and attributes. |
666 |
# |
# |
667 |
$HardLinkExtraFiles{$hdr->{linkname}} = 1; |
$HardLinkExtraFiles{$hdr->{linkname}} = 1; |
668 |
archiveWrite($fh, $hdr->{linkname}, $hdr->{name}); |
archiveWrite($fh, $hdr->{linkname}, $hdr->{name}); |
669 |
} |
} |
670 |
} elsif ( $hdr->{type} == BPC_FTYPE_SYMLINK ) { |
} elsif ( $hdr->{type} == BPC_FTYPE_SYMLINK ) { |
671 |
# |
# |
672 |
# Symbolic link: read the symbolic link contents into the header |
# Symbolic link: read the symbolic link contents into the header |