1 |
#!/usr/bin/perl |
#!/usr/bin/perl -w |
2 |
#============================================================= -*-perl-*- |
#============================================================= -*-perl-*- |
3 |
# |
# |
4 |
# BackupPC_tarIncCreate: create a tar archive of an existing incremental dump |
# BackupPC_tarIncCreate: create a tar archive of an existing incremental dump |
72 |
use BackupPC::SearchLib; |
use BackupPC::SearchLib; |
73 |
use Time::HiRes qw/time/; |
use Time::HiRes qw/time/; |
74 |
use POSIX qw/strftime/; |
use POSIX qw/strftime/; |
75 |
|
use File::Which; |
76 |
|
use File::Path; |
77 |
|
use File::Slurp; |
78 |
use Data::Dumper; ### FIXME |
use Data::Dumper; ### FIXME |
79 |
|
|
80 |
die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); |
die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); |
81 |
my $TopDir = $bpc->TopDir(); |
my $TopDir = $bpc->TopDir(); |
82 |
my $BinDir = $bpc->BinDir(); |
my $BinDir = $bpc->BinDir(); |
83 |
my %Conf = $bpc->Conf(); |
my %Conf = $bpc->Conf(); |
84 |
|
%BackupPC::SearchLib::Conf = %Conf; |
85 |
my %opts; |
my %opts; |
86 |
my $in_backup_increment; |
my $in_backup_increment; |
87 |
|
|
88 |
|
|
89 |
if ( !getopts("th:n:p:r:s:b:w:v", \%opts) ) { |
if ( !getopts("th:n:p:r:s:b:w:vd", \%opts) ) { |
90 |
print STDERR <<EOF; |
print STDERR <<EOF; |
91 |
usage: $0 [options] |
usage: $0 [options] |
92 |
Required options: |
Required options: |
103 |
-b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar) |
-b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar) |
104 |
-w writeBufSz write buffer size (default 1048576 = 1MB) |
-w writeBufSz write buffer size (default 1048576 = 1MB) |
105 |
-v verbose output |
-v verbose output |
106 |
|
-d debug output |
107 |
EOF |
EOF |
108 |
exit(1); |
exit(1); |
109 |
} |
} |
120 |
} |
} |
121 |
my $Num = $opts{n}; |
my $Num = $opts{n}; |
122 |
|
|
123 |
|
my $bin; |
124 |
|
foreach my $c (qw/gzip md5sum tee/) { |
125 |
|
$bin->{$c} = which($c) || die "$0 needs $c, install it\n"; |
126 |
|
} |
127 |
|
|
128 |
my @Backups = $bpc->BackupInfoRead($Host); |
my @Backups = $bpc->BackupInfoRead($Host); |
129 |
my $FileCnt = 0; |
my $FileCnt = 0; |
130 |
my $ByteCnt = 0; |
my $ByteCnt = 0; |
131 |
my $DirCnt = 0; |
my $DirCnt = 0; |
132 |
my $SpecialCnt = 0; |
my $SpecialCnt = 0; |
133 |
my $ErrorCnt = 0; |
my $ErrorCnt = 0; |
134 |
|
my $current_tar_size = 0; |
135 |
|
|
136 |
my $i; |
my $i; |
137 |
$Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 ); |
$Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 ); |
152 |
our $ShareName = $opts{s}; |
our $ShareName = $opts{s}; |
153 |
our $view = BackupPC::View->new($bpc, $Host, \@Backups); |
our $view = BackupPC::View->new($bpc, $Host, \@Backups); |
154 |
|
|
155 |
|
# database |
156 |
|
|
157 |
|
my $dsn = $Conf{SearchDSN}; |
158 |
|
my $db_user = $Conf{SearchUser} || ''; |
159 |
|
|
160 |
|
my $dbh = DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 0} ); |
161 |
|
|
162 |
|
my $sth_inc_size = $dbh->prepare(qq{ |
163 |
|
update backups set |
164 |
|
inc_size = ?, |
165 |
|
parts = ?, |
166 |
|
inc_deleted = false |
167 |
|
where id = ? }); |
168 |
|
my $sth_backup_parts = $dbh->prepare(qq{ |
169 |
|
insert into backup_parts ( |
170 |
|
backup_id, |
171 |
|
part_nr, |
172 |
|
tar_size, |
173 |
|
size, |
174 |
|
md5, |
175 |
|
items |
176 |
|
) values (?,?,?,?,?,?) |
177 |
|
}); |
178 |
|
|
179 |
# |
# |
180 |
# This constant and the line of code below that uses it are borrowed |
# This constant and the line of code below that uses it are borrowed |
181 |
# from Archive::Tar. Thanks to Calle Dybedahl and Stephen Zander. |
# from Archive::Tar. Thanks to Calle Dybedahl and Stephen Zander. |
198 |
# |
# |
199 |
# Write out all the requested files/directories |
# Write out all the requested files/directories |
200 |
# |
# |
201 |
binmode(STDOUT); |
|
202 |
my $fh = *STDOUT; |
my $max_file_size = $Conf{'MaxArchiveFileSize'} || die "problem with MaxArchiveFileSize parametar"; |
203 |
|
$max_file_size *= 1024; |
204 |
|
|
205 |
|
my $tar_dir = $Conf{InstallDir}.'/'.$Conf{GzipTempDir}; |
206 |
|
die "problem with $tar_dir, check GzipTempDir in configuration\n" unless (-d $tar_dir && -w $tar_dir); |
207 |
|
|
208 |
|
my $tar_file = BackupPC::SearchLib::getGzipName($Host, $ShareName, $Num) || die "can't getGzipName($Host, $ShareName, $Num)"; |
209 |
|
|
210 |
|
my $tar_path = $tar_dir . '/' . $tar_file . '.tmp'; |
211 |
|
$tar_path =~ s#//#/#g; |
212 |
|
|
213 |
|
my $sth = $dbh->prepare(qq{ |
214 |
|
SELECT |
215 |
|
backups.id |
216 |
|
FROM backups |
217 |
|
JOIN shares on shares.id = shareid |
218 |
|
JOIN hosts on hosts.id = shares.hostid |
219 |
|
WHERE hosts.name = ? and shares.name = ? and backups.num = ? |
220 |
|
}); |
221 |
|
$sth->execute($Host, $ShareName, $Num); |
222 |
|
my ($backup_id) = $sth->fetchrow_array; |
223 |
|
$sth->finish; |
224 |
|
|
225 |
|
print STDERR "backup_id: $backup_id working dir: $tar_dir, max uncompressed size $max_file_size bytes, tar $tar_file\n" if ($opts{d}); |
226 |
|
|
227 |
|
|
228 |
|
my $fh; |
229 |
|
my $part = 0; |
230 |
|
my $no_files = 0; |
231 |
|
my $items_in_part = 0; |
232 |
|
|
233 |
|
sub new_tar_part { |
234 |
|
if ($fh) { |
235 |
|
return if ($current_tar_size == 0); |
236 |
|
|
237 |
|
print STDERR "# closing part $part\n" if ($opts{d}); |
238 |
|
|
239 |
|
# finish tar archive |
240 |
|
my $data = "\0" x ($tar_header_length * 2); |
241 |
|
TarWrite($fh, \$data); |
242 |
|
TarWrite($fh, undef); |
243 |
|
|
244 |
|
close($fh) || die "can't close archive part $part: $!"; |
245 |
|
|
246 |
|
my $file = $tar_path . '/' . $part; |
247 |
|
|
248 |
|
my $md5 = read_file( $file . '.md5' ) || die "can't read md5sum file ${file}.md5"; |
249 |
|
$md5 =~ s/\s.*$//; |
250 |
|
|
251 |
|
my $size = (stat( $file . '.tar.gz' ))[7] || die "can't stat ${file}.tar.gz"; |
252 |
|
|
253 |
|
$sth_backup_parts->execute( |
254 |
|
$backup_id, |
255 |
|
$part, |
256 |
|
$current_tar_size, |
257 |
|
$size, |
258 |
|
$md5, |
259 |
|
$items_in_part, |
260 |
|
); |
261 |
|
|
262 |
|
} |
263 |
|
|
264 |
|
$part++; |
265 |
|
|
266 |
|
# if this is first part, create directory |
267 |
|
|
268 |
|
if ($part == 1) { |
269 |
|
if (-d $tar_path) { |
270 |
|
print STDERR "# deleting existing $tar_path\n" if ($opts{d}); |
271 |
|
rmtree($tar_path); |
272 |
|
} |
273 |
|
mkdir($tar_path) || die "can't create directory $tar_path: $!"; |
274 |
|
} |
275 |
|
|
276 |
|
my $file = $tar_path . '/' . $part; |
277 |
|
|
278 |
|
# |
279 |
|
# create comprex pipe which will pass output through gzip |
280 |
|
# for compression, create file on disk using tee |
281 |
|
# and pipe same output to md5sum to create checksum |
282 |
|
# |
283 |
|
|
284 |
|
my $cmd = '| ' . $bin->{'gzip'} . ' ' . $Conf{GzipLevel} . ' ' . |
285 |
|
'| ' . $bin->{'tee'} . ' ' . $file . '.tar.gz' . ' ' . |
286 |
|
'| ' . $bin->{'md5sum'} . ' - > ' . $file . '.md5'; |
287 |
|
|
288 |
|
print STDERR "## $cmd\n" if ($opts{d}); |
289 |
|
|
290 |
|
open($fh, $cmd) or die "can't open $cmd: $!"; |
291 |
|
binmode($fh); |
292 |
|
|
293 |
|
$current_tar_size = 0; |
294 |
|
$items_in_part = 0; |
295 |
|
} |
296 |
|
|
297 |
|
new_tar_part(); |
298 |
|
|
299 |
if (seedCache($Host, $ShareName, $Num)) { |
if (seedCache($Host, $ShareName, $Num)) { |
300 |
archiveWrite($fh, '/'); |
archiveWrite($fh, '/'); |
301 |
archiveWriteHardLinks($fh); |
archiveWriteHardLinks($fh); |
302 |
} else { |
} else { |
303 |
print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n"; |
print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n" if ($opts{v}); |
304 |
|
$no_files = 1; |
305 |
} |
} |
306 |
|
|
307 |
# |
# |
312 |
TarWrite($fh, \$data); |
TarWrite($fh, \$data); |
313 |
TarWrite($fh, undef); |
TarWrite($fh, undef); |
314 |
|
|
315 |
|
if (! close($fh)) { |
316 |
|
rmtree($tar_path); |
317 |
|
die "can't close archive\n"; |
318 |
|
} |
319 |
|
|
320 |
|
# remove temporary files if there are no files |
321 |
|
if ($no_files) { |
322 |
|
rmtree($tar_path); |
323 |
|
} elsif ($part == 1) { |
324 |
|
warn "FIXME: if there is only one part move to parent directory and rename"; |
325 |
|
} |
326 |
|
|
327 |
# |
# |
328 |
# print out totals if requested |
# print out totals if requested |
329 |
# |
# |
336 |
# Got errors, with no files or directories; exit with non-zero |
# Got errors, with no files or directories; exit with non-zero |
337 |
# status |
# status |
338 |
# |
# |
339 |
|
cleanup(); |
340 |
exit(1); |
exit(1); |
341 |
} |
} |
342 |
|
|
343 |
|
$sth_inc_size->finish; |
344 |
|
$sth_backup_parts->finish; |
345 |
|
|
346 |
|
$dbh->commit || die "can't commit changes to database"; |
347 |
|
$dbh->disconnect(); |
348 |
|
|
349 |
exit(0); |
exit(0); |
350 |
|
|
351 |
########################################################################### |
########################################################################### |
411 |
{ |
{ |
412 |
my($fh, $dataRef) = @_; |
my($fh, $dataRef) = @_; |
413 |
|
|
414 |
|
|
415 |
if ( !defined($dataRef) ) { |
if ( !defined($dataRef) ) { |
416 |
# |
# |
417 |
# do flush by padding to a full $WriteBufSz |
# do flush by padding to a full $WriteBufSz |
419 |
my $data = "\0" x ($WriteBufSz - length($WriteBuf)); |
my $data = "\0" x ($WriteBufSz - length($WriteBuf)); |
420 |
$dataRef = \$data; |
$dataRef = \$data; |
421 |
} |
} |
422 |
|
|
423 |
|
# poor man's tell :-) |
424 |
|
$current_tar_size += length($$dataRef); |
425 |
|
|
426 |
if ( length($WriteBuf) + length($$dataRef) < $WriteBufSz ) { |
if ( length($WriteBuf) + length($$dataRef) < $WriteBufSz ) { |
427 |
# |
# |
428 |
# just buffer and return |
# just buffer and return |
545 |
sub seedCache($$$) { |
sub seedCache($$$) { |
546 |
my ($host, $share, $dumpNo) = @_; |
my ($host, $share, $dumpNo) = @_; |
547 |
|
|
|
my $dsn = $Conf{SearchDSN}; |
|
|
my $db_user = $Conf{SearchUser} || ''; |
|
|
|
|
548 |
print STDERR curr_time(), "getting files for $host:$share increment $dumpNo..." if ($opts{v}); |
print STDERR curr_time(), "getting files for $host:$share increment $dumpNo..." if ($opts{v}); |
549 |
my $sql = q{ |
my $sql = q{ |
550 |
SELECT path |
SELECT path,size |
551 |
FROM files |
FROM files |
552 |
JOIN shares on shares.id = shareid |
JOIN shares on shares.id = shareid |
553 |
JOIN hosts on hosts.id = shares.hostid |
JOIN hosts on hosts.id = shares.hostid |
554 |
WHERE hosts.name = ? and shares.name = ? and backupnum = ? |
WHERE hosts.name = ? and shares.name = ? and backupnum = ? |
555 |
}; |
}; |
556 |
|
|
|
my $dbh = DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1} ); |
|
557 |
my $sth = $dbh->prepare($sql); |
my $sth = $dbh->prepare($sql); |
558 |
$sth->execute($host, $share, $dumpNo); |
$sth->execute($host, $share, $dumpNo); |
559 |
my $count = $sth->rows; |
my $count = $sth->rows; |
560 |
print STDERR " found $count items\n" if ($opts{v}); |
print STDERR " found $count items\n" if ($opts{v}); |
561 |
while (my $row = $sth->fetchrow_arrayref) { |
while (my $row = $sth->fetchrow_arrayref) { |
562 |
#print STDERR "+ ", $row->[0],"\n"; |
#print STDERR "+ ", $row->[0],"\n"; |
563 |
$in_backup_increment->{ $row->[0] }++; |
$in_backup_increment->{ $row->[0] } = $row->[1]; |
564 |
} |
} |
565 |
|
|
566 |
$sth->finish(); |
$sth->finish(); |
|
$dbh->disconnect(); |
|
567 |
|
|
568 |
return $count; |
return $count; |
569 |
} |
} |
570 |
|
|
571 |
|
# |
572 |
|
# calculate overhad for one file in tar |
573 |
|
# |
574 |
|
sub tar_overhead($) { |
575 |
|
my $name = shift || ''; |
576 |
|
|
577 |
|
# header, padding of file and two null blocks at end |
578 |
|
my $len = 4 * $tar_header_length; |
579 |
|
|
580 |
|
# if filename is longer than 99 chars subtract blocks for |
581 |
|
# long filename |
582 |
|
if ( length($name) > 99 ) { |
583 |
|
$len += int( ( length($name) + $tar_header_length ) / $tar_header_length ) * $tar_header_length; |
584 |
|
} |
585 |
|
|
586 |
|
return $len; |
587 |
|
} |
588 |
|
|
589 |
my $Attr; |
my $Attr; |
590 |
my $AttrDir; |
my $AttrDir; |
591 |
|
|
598 |
|
|
599 |
$tarPath =~ s{//+}{/}g; |
$tarPath =~ s{//+}{/}g; |
600 |
|
|
601 |
#print STDERR "? $tarPath\n"; |
#print STDERR "? $tarPath\n" if ($opts{d}); |
602 |
return unless ($in_backup_increment->{$tarPath}); |
my $size = $in_backup_increment->{$tarPath}; |
603 |
#print STDERR "A $tarPath\n"; |
return unless (defined($size)); |
604 |
|
|
605 |
|
# is this file too large to fit into MaxArchiveFileSize? |
606 |
|
|
607 |
|
if ( ($current_tar_size + tar_overhead($tarPath) + $size) > $max_file_size ) { |
608 |
|
print STDERR "# tar file $current_tar_size + $tar_header_length + $size > $max_file_size, splitting\n" if ($opts{d}); |
609 |
|
new_tar_part(); |
610 |
|
} |
611 |
|
|
612 |
|
#print STDERR "A $tarPath [$size] tell: $current_tar_size\n" if ($opts{d}); |
613 |
|
$items_in_part++; |
614 |
|
|
615 |
if ( defined($PathRemove) |
if ( defined($PathRemove) |
616 |
&& substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) { |
&& substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) { |
624 |
# |
# |
625 |
# Directory: just write the header |
# Directory: just write the header |
626 |
# |
# |
|
|
|
|
|
|
627 |
$hdr->{name} .= "/" if ( $hdr->{name} !~ m{/$} ); |
$hdr->{name} .= "/" if ( $hdr->{name} !~ m{/$} ); |
628 |
TarWriteFileInfo($fh, $hdr); |
TarWriteFileInfo($fh, $hdr); |
629 |
$DirCnt++; |
$DirCnt++; |
637 |
$ErrorCnt++; |
$ErrorCnt++; |
638 |
return; |
return; |
639 |
} |
} |
640 |
TarWriteFileInfo($fh, $hdr); |
# do we need to split file? |
641 |
my($data, $size); |
if ($hdr->{size} < $max_file_size) { |
642 |
while ( $f->read(\$data, $BufSize) > 0 ) { |
TarWriteFileInfo($fh, $hdr); |
643 |
TarWrite($fh, \$data); |
my($data, $size); |
644 |
$size += length($data); |
while ( $f->read(\$data, $BufSize) > 0 ) { |
645 |
} |
TarWrite($fh, \$data); |
646 |
$f->close; |
$size += length($data); |
647 |
TarWritePad($fh, $size); |
} |
648 |
|
$f->close; |
649 |
|
TarWritePad($fh, $size); |
650 |
$FileCnt++; |
$FileCnt++; |
651 |
$ByteCnt += $size; |
$ByteCnt += $size; |
652 |
|
} else { |
653 |
|
my $full_size = $hdr->{size}; |
654 |
|
my $orig_name = $hdr->{name}; |
655 |
|
my $max_part_size = $max_file_size - tar_overhead($hdr->{name}); |
656 |
|
|
657 |
|
my $parts = int(($full_size + $max_part_size - 1) / $max_part_size); |
658 |
|
print STDERR "# splitting $orig_name [$full_size bytes] into $parts parts\n" if ($opts{d}); |
659 |
|
foreach my $subpart ( 1 .. $parts ) { |
660 |
|
new_tar_part(); |
661 |
|
if ($subpart < $parts) { |
662 |
|
$hdr->{size} = $max_part_size; |
663 |
|
} else { |
664 |
|
$hdr->{size} = $full_size % $max_part_size; |
665 |
|
} |
666 |
|
$hdr->{name} = $orig_name . '/' . $subpart; |
667 |
|
print STDERR "## creating part $subpart ",$hdr->{name}, " [", $hdr->{size}," bytes]\n"; |
668 |
|
|
669 |
|
TarWriteFileInfo($fh, $hdr); |
670 |
|
my($data, $size); |
671 |
|
if (0) { |
672 |
|
for ( 1 .. int($hdr->{size} / $BufSize) ) { |
673 |
|
my $r_size = $f->read(\$data, $BufSize); |
674 |
|
die "expected $BufSize bytes read, got $r_size bytes!" if ($r_size != $BufSize); |
675 |
|
TarWrite($fh, \$data); |
676 |
|
$size += length($data); |
677 |
|
} |
678 |
|
} |
679 |
|
my $size_left = $hdr->{size} % $BufSize; |
680 |
|
my $r_size = $f->read(\$data, $size_left); |
681 |
|
die "expected $size_left bytes last read, got $r_size bytes!" if ($r_size != $size_left); |
682 |
|
|
683 |
|
TarWrite($fh, \$data); |
684 |
|
$size += length($data); |
685 |
|
TarWritePad($fh, $size); |
686 |
|
|
687 |
|
$items_in_part++; |
688 |
|
} |
689 |
|
$f->close; |
690 |
|
$FileCnt++; |
691 |
|
$ByteCnt += $full_size; |
692 |
|
new_tar_part(); |
693 |
|
} |
694 |
} elsif ( $hdr->{type} == BPC_FTYPE_HARDLINK ) { |
} elsif ( $hdr->{type} == BPC_FTYPE_HARDLINK ) { |
695 |
# |
# |
696 |
# Hardlink file: either write a hardlink or the complete file |
# Hardlink file: either write a hardlink or the complete file |
697 |
# depending upon whether the linked-to file will be written |
# depending upon whether the linked-to file will be written |
698 |
# to the archive. |
# to the archive. |
699 |
# |
# |
700 |
# Start by reading the contents of the link. |
# Start by reading the contents of the link. |
701 |
# |
# |
702 |
my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress}); |
my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress}); |
703 |
if ( !defined($f) ) { |
if ( !defined($f) ) { |
704 |
print(STDERR "Unable to open file $hdr->{fullPath}\n"); |
print(STDERR "Unable to open file $hdr->{fullPath}\n"); |
709 |
while ( $f->read(\$data, $BufSize) > 0 ) { |
while ( $f->read(\$data, $BufSize) > 0 ) { |
710 |
$hdr->{linkname} .= $data; |
$hdr->{linkname} .= $data; |
711 |
} |
} |
712 |
$f->close; |
$f->close; |
713 |
my $done = 0; |
my $done = 0; |
714 |
my $name = $hdr->{linkname}; |
my $name = $hdr->{linkname}; |
715 |
$name =~ s{^\./}{/}; |
$name =~ s{^\./}{/}; |
716 |
if ( $HardLinkExtraFiles{$name} ) { |
if ( $HardLinkExtraFiles{$name} ) { |
717 |
# |
# |
718 |
# Target file will be or was written, so just remember |
# Target file will be or was written, so just remember |
719 |
# the hardlink so we can dump it later. |
# the hardlink so we can dump it later. |
720 |
# |
# |
721 |
push(@HardLinks, $hdr); |
push(@HardLinks, $hdr); |
722 |
$SpecialCnt++; |
$SpecialCnt++; |
723 |
} else { |
} else { |
724 |
# |
# |
725 |
# Have to dump the original file. Just call the top-level |
# Have to dump the original file. Just call the top-level |
726 |
# routine, so that we save the hassle of dealing with |
# routine, so that we save the hassle of dealing with |
727 |
# mangling, merging and attributes. |
# mangling, merging and attributes. |
728 |
# |
# |
729 |
$HardLinkExtraFiles{$hdr->{linkname}} = 1; |
$HardLinkExtraFiles{$hdr->{linkname}} = 1; |
730 |
archiveWrite($fh, $hdr->{linkname}, $hdr->{name}); |
archiveWrite($fh, $hdr->{linkname}, $hdr->{name}); |
731 |
} |
} |
732 |
} elsif ( $hdr->{type} == BPC_FTYPE_SYMLINK ) { |
} elsif ( $hdr->{type} == BPC_FTYPE_SYMLINK ) { |
733 |
# |
# |
734 |
# Symbolic link: read the symbolic link contents into the header |
# Symbolic link: read the symbolic link contents into the header |