74 |
use POSIX qw/strftime/; |
use POSIX qw/strftime/; |
75 |
use File::Which; |
use File::Which; |
76 |
use File::Path; |
use File::Path; |
77 |
|
use File::Slurp; |
78 |
use Data::Dumper; ### FIXME |
use Data::Dumper; ### FIXME |
79 |
|
|
80 |
die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); |
die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) ); |
86 |
my $in_backup_increment; |
my $in_backup_increment; |
87 |
|
|
88 |
|
|
89 |
if ( !getopts("th:n:p:r:s:b:w:vd", \%opts) ) { |
if ( !getopts("th:n:p:r:s:b:w:vdf", \%opts) ) { |
90 |
print STDERR <<EOF; |
print STDERR <<EOF; |
91 |
usage: $0 [options] |
usage: $0 [options] |
92 |
Required options: |
Required options: |
102 |
-p pathAdd new path prefix |
-p pathAdd new path prefix |
103 |
-b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar) |
-b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar) |
104 |
-w writeBufSz write buffer size (default 1048576 = 1MB) |
-w writeBufSz write buffer size (default 1048576 = 1MB) |
105 |
|
-f overwrite existing parts |
106 |
-v verbose output |
-v verbose output |
107 |
-d debug output |
-d debug output |
108 |
EOF |
EOF |
110 |
} |
} |
111 |
|
|
112 |
if ( $opts{h} !~ /^([\w\.\s-]+)$/ ) { |
if ( $opts{h} !~ /^([\w\.\s-]+)$/ ) { |
113 |
print(STDERR "$0: bad host name '$opts{h}'\n"); |
die "$0: bad host name '$opts{h}'\n"; |
|
exit(1); |
|
114 |
} |
} |
115 |
my $Host = $opts{h}; |
my $Host = $opts{h}; |
116 |
|
|
117 |
if ( $opts{n} !~ /^(-?\d+)$/ ) { |
if ( $opts{n} !~ /^(-?\d+)$/ ) { |
118 |
print(STDERR "$0: bad dump number '$opts{n}'\n"); |
die "$0: bad dump number '$opts{n}'\n"; |
|
exit(1); |
|
119 |
} |
} |
120 |
my $Num = $opts{n}; |
my $Num = $opts{n}; |
121 |
|
|
131 |
my $SpecialCnt = 0; |
my $SpecialCnt = 0; |
132 |
my $ErrorCnt = 0; |
my $ErrorCnt = 0; |
133 |
my $current_tar_size = 0; |
my $current_tar_size = 0; |
134 |
|
my $total_increment_size = 0; |
135 |
|
|
136 |
my $i; |
my $i; |
137 |
$Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 ); |
$Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 ); |
139 |
last if ( $Backups[$i]{num} == $Num ); |
last if ( $Backups[$i]{num} == $Num ); |
140 |
} |
} |
141 |
if ( $i >= @Backups ) { |
if ( $i >= @Backups ) { |
142 |
print(STDERR "$0: bad backup number $Num for host $Host\n"); |
die "$0: bad backup number $Num for host $Host\n"; |
|
exit(1); |
|
143 |
} |
} |
144 |
|
|
145 |
my $PathRemove = $1 if ( $opts{r} =~ /(.+)/ ); |
my $PathRemove = $1 if ( $opts{r} =~ /(.+)/ ); |
146 |
my $PathAdd = $1 if ( $opts{p} =~ /(.+)/ ); |
my $PathAdd = $1 if ( $opts{p} =~ /(.+)/ ); |
147 |
if ( $opts{s} !~ /^([\w\s\.\/\$-]+)$/ && $opts{s} ne "*" ) { |
if ( $opts{s} !~ /^([\w\s\.\/\$-]+)$/ && $opts{s} ne "*" ) { |
148 |
print(STDERR "$0: bad share name '$opts{s}'\n"); |
die "$0: bad share name '$opts{s}'\n"; |
|
exit(1); |
|
149 |
} |
} |
150 |
our $ShareName = $opts{s}; |
our $ShareName = $opts{s}; |
151 |
our $view = BackupPC::View->new($bpc, $Host, \@Backups); |
our $view = BackupPC::View->new($bpc, $Host, \@Backups); |
152 |
|
|
153 |
|
# database |
154 |
|
|
155 |
|
my $dsn = $Conf{SearchDSN}; |
156 |
|
my $db_user = $Conf{SearchUser} || ''; |
157 |
|
|
158 |
|
my $dbh = DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 0} ); |
159 |
|
|
160 |
|
my $sth_inc_size = $dbh->prepare(qq{ |
161 |
|
update backups set |
162 |
|
inc_size = ?, |
163 |
|
parts = ?, |
164 |
|
inc_deleted = false |
165 |
|
where id = ? |
166 |
|
}); |
167 |
|
my $sth_backup_parts = $dbh->prepare(qq{ |
168 |
|
insert into backup_parts ( |
169 |
|
backup_id, |
170 |
|
part_nr, |
171 |
|
tar_size, |
172 |
|
size, |
173 |
|
md5, |
174 |
|
items |
175 |
|
) values (?,?,?,?,?,?) |
176 |
|
}); |
177 |
|
|
178 |
# |
# |
179 |
# This constant and the line of code below that uses it are borrowed |
# This constant and the line of code below that uses it are borrowed |
180 |
# from Archive::Tar. Thanks to Calle Dybedahl and Stephen Zander. |
# from Archive::Tar. Thanks to Calle Dybedahl and Stephen Zander. |
206 |
|
|
207 |
my $tar_file = BackupPC::SearchLib::getGzipName($Host, $ShareName, $Num) || die "can't getGzipName($Host, $ShareName, $Num)"; |
my $tar_file = BackupPC::SearchLib::getGzipName($Host, $ShareName, $Num) || die "can't getGzipName($Host, $ShareName, $Num)"; |
208 |
|
|
209 |
my $tar_path = $tar_dir . '/' . $tar_file . '.tmp'; |
my $tar_path_final = $tar_dir . '/' . $tar_file; |
210 |
|
my $tar_path = $tar_path_final . '.tmp'; |
211 |
|
|
212 |
$tar_path =~ s#//#/#g; |
$tar_path =~ s#//#/#g; |
213 |
|
|
214 |
print STDERR "working dir: $tar_dir, max uncompressed size $max_file_size bytes, tar $tar_file\n" if ($opts{d}); |
my $sth = $dbh->prepare(qq{ |
215 |
|
SELECT |
216 |
|
backups.id |
217 |
|
FROM backups |
218 |
|
JOIN shares on shares.id = shareid |
219 |
|
JOIN hosts on hosts.id = shares.hostid |
220 |
|
WHERE hosts.name = ? and shares.name = ? and backups.num = ? |
221 |
|
}); |
222 |
|
$sth->execute($Host, $ShareName, $Num); |
223 |
|
my ($backup_id) = $sth->fetchrow_array; |
224 |
|
$sth->finish; |
225 |
|
|
226 |
|
print STDERR "backup_id: $backup_id working dir: $tar_dir, max uncompressed size $max_file_size bytes, tar $tar_file\n" if ($opts{d}); |
227 |
|
|
228 |
|
if (-e $tar_path_final) { |
229 |
|
if ($opts{f}) { |
230 |
|
rmtree $tar_path_final || die "can't remove $tar_path_final: $!"; |
231 |
|
} else { |
232 |
|
die "$tar_path_final allready exists\n"; |
233 |
|
} |
234 |
|
} |
235 |
|
|
236 |
my $fh; |
my $fh; |
237 |
my $part = 0; |
my $part = 0; |
238 |
my $no_files = 0; |
my $no_files = 0; |
239 |
|
my $items_in_part = 0; |
240 |
|
|
241 |
sub new_tar_part { |
sub new_tar_part { |
242 |
|
my $arg = {@_}; |
243 |
|
|
244 |
if ($fh) { |
if ($fh) { |
245 |
return if ($current_tar_size == 0); |
return if ($current_tar_size == 0); |
246 |
|
|
247 |
print STDERR "# closing part $part\n" if ($opts{d}); |
print STDERR " $part" if ($opts{v}); |
248 |
|
|
249 |
# finish tar archive |
# |
250 |
|
# Finish with two null 512 byte headers, |
251 |
|
# and then round out a full block. |
252 |
|
# |
253 |
my $data = "\0" x ($tar_header_length * 2); |
my $data = "\0" x ($tar_header_length * 2); |
254 |
TarWrite($fh, \$data); |
TarWrite($fh, \$data); |
255 |
TarWrite($fh, undef); |
TarWrite($fh, undef); |
256 |
|
|
257 |
close($fh) || die "can't close archive part $part: $!"; |
close($fh) || die "can't close archive part $part: $!"; |
258 |
|
|
259 |
|
my $file = $tar_path . '/' . $part; |
260 |
|
|
261 |
|
my $md5 = read_file( $file . '.md5' ) || die "can't read md5sum file ${file}.md5"; |
262 |
|
$md5 =~ s/\s.*$//; |
263 |
|
|
264 |
|
my $size = (stat( $file . '.tar.gz' ))[7] || die "can't stat ${file}.tar.gz"; |
265 |
|
|
266 |
|
$sth_backup_parts->execute( |
267 |
|
$backup_id, |
268 |
|
$part, |
269 |
|
$current_tar_size, |
270 |
|
$size, |
271 |
|
$md5, |
272 |
|
$items_in_part, |
273 |
|
); |
274 |
|
|
275 |
|
$total_increment_size += int( ( $size + 1023 ) / 1024 ) * 1024; |
276 |
|
|
277 |
|
if ($arg->{close}) { |
278 |
|
|
279 |
|
sub move($$) { |
280 |
|
my ($from,$to) = @_; |
281 |
|
print STDERR "# rename $from -> $to\n" if ($opts{d}); |
282 |
|
rename $from, $to || die "can't move $from -> $to: $!\n"; |
283 |
|
} |
284 |
|
|
285 |
|
if ($part == 1) { |
286 |
|
print STDERR " single"; |
287 |
|
move("${tar_path}/1.tar.gz", "${tar_path_final}.tar.gz"); |
288 |
|
move("${tar_path}/1.md5", "${tar_path_final}.md5"); |
289 |
|
rmtree $tar_path or die "can't remove temporary dir $tar_path: $!"; |
290 |
|
} else { |
291 |
|
print STDERR " [last]"; |
292 |
|
move("${tar_path}", "${tar_path_final}"); |
293 |
|
} |
294 |
|
|
295 |
|
$sth_inc_size->execute( |
296 |
|
$total_increment_size, |
297 |
|
$part, |
298 |
|
$backup_id |
299 |
|
); |
300 |
|
|
301 |
|
print STDERR ", $total_increment_size bytes\n" if ($opts{v}); |
302 |
|
|
303 |
|
return; |
304 |
|
} |
305 |
|
|
306 |
} |
} |
307 |
|
|
308 |
$part++; |
$part++; |
315 |
rmtree($tar_path); |
rmtree($tar_path); |
316 |
} |
} |
317 |
mkdir($tar_path) || die "can't create directory $tar_path: $!"; |
mkdir($tar_path) || die "can't create directory $tar_path: $!"; |
318 |
|
|
319 |
|
sub abort_cleanup { |
320 |
|
print STDERR "ABORTED: cleanup temp dir"; |
321 |
|
rmtree($tar_path); |
322 |
|
$dbh->rollback; |
323 |
|
exit 1; |
324 |
|
} |
325 |
|
|
326 |
|
$SIG{'INT'} = \&abort_cleanup; |
327 |
|
$SIG{'QUIT'} = \&abort_cleanup; |
328 |
|
$SIG{'__DIE__'} = \&abort_cleanup; |
329 |
|
|
330 |
} |
} |
331 |
|
|
332 |
my $file = $tar_path . '/' . $part; |
my $file = $tar_path . '/' . $part; |
345 |
|
|
346 |
open($fh, $cmd) or die "can't open $cmd: $!"; |
open($fh, $cmd) or die "can't open $cmd: $!"; |
347 |
binmode($fh); |
binmode($fh); |
348 |
|
|
349 |
$current_tar_size = 0; |
$current_tar_size = 0; |
350 |
|
$items_in_part = 0; |
351 |
} |
} |
352 |
|
|
353 |
new_tar_part(); |
new_tar_part(); |
355 |
if (seedCache($Host, $ShareName, $Num)) { |
if (seedCache($Host, $ShareName, $Num)) { |
356 |
archiveWrite($fh, '/'); |
archiveWrite($fh, '/'); |
357 |
archiveWriteHardLinks($fh); |
archiveWriteHardLinks($fh); |
358 |
|
new_tar_part( close => 1 ); |
359 |
} else { |
} else { |
360 |
print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n" if ($opts{v}); |
print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n" if ($opts{v}); |
361 |
$no_files = 1; |
# remove temporary files if there are no files |
|
} |
|
|
|
|
|
# |
|
|
# Finish with two null 512 byte headers, and then round out a full |
|
|
# block. |
|
|
# |
|
|
my $data = "\0" x ($tar_header_length * 2); |
|
|
TarWrite($fh, \$data); |
|
|
TarWrite($fh, undef); |
|
|
|
|
|
if (! close($fh)) { |
|
|
rmtree($tar_path); |
|
|
die "can't close archive\n"; |
|
|
} |
|
|
|
|
|
# remove temporary files if there are no files |
|
|
if ($no_files) { |
|
362 |
rmtree($tar_path); |
rmtree($tar_path); |
|
} elsif ($part == 1) { |
|
|
warn "FIXME: if there is only one part move to parent directory and rename"; |
|
363 |
} |
} |
364 |
|
|
365 |
# |
# |
374 |
# Got errors, with no files or directories; exit with non-zero |
# Got errors, with no files or directories; exit with non-zero |
375 |
# status |
# status |
376 |
# |
# |
377 |
cleanup(); |
die "got errors or no files\n"; |
|
exit(1); |
|
378 |
} |
} |
379 |
|
|
380 |
exit(0); |
$sth_inc_size->finish; |
381 |
|
$sth_backup_parts->finish; |
382 |
|
|
383 |
|
$dbh->commit || die "can't commit changes to database"; |
384 |
|
$dbh->disconnect(); |
385 |
|
|
386 |
|
exit; |
387 |
|
|
388 |
########################################################################### |
########################################################################### |
389 |
# Subroutines |
# Subroutines |
470 |
my $done = $WriteBufSz - length($WriteBuf); |
my $done = $WriteBufSz - length($WriteBuf); |
471 |
if ( syswrite($fh, $WriteBuf . substr($$dataRef, 0, $done)) |
if ( syswrite($fh, $WriteBuf . substr($$dataRef, 0, $done)) |
472 |
!= $WriteBufSz ) { |
!= $WriteBufSz ) { |
473 |
print(STDERR "Unable to write to output file ($!)\n"); |
die "Unable to write to output file ($!)\n"; |
|
exit(1); |
|
474 |
} |
} |
475 |
while ( $done + $WriteBufSz <= length($$dataRef) ) { |
while ( $done + $WriteBufSz <= length($$dataRef) ) { |
476 |
if ( syswrite($fh, substr($$dataRef, $done, $WriteBufSz)) |
if ( syswrite($fh, substr($$dataRef, $done, $WriteBufSz)) |
477 |
!= $WriteBufSz ) { |
!= $WriteBufSz ) { |
478 |
print(STDERR "Unable to write to output file ($!)\n"); |
die "Unable to write to output file ($!)\n"; |
|
exit(1); |
|
479 |
} |
} |
480 |
$done += $WriteBufSz; |
$done += $WriteBufSz; |
481 |
} |
} |
580 |
sub seedCache($$$) { |
sub seedCache($$$) { |
581 |
my ($host, $share, $dumpNo) = @_; |
my ($host, $share, $dumpNo) = @_; |
582 |
|
|
583 |
my $dsn = $Conf{SearchDSN}; |
print STDERR curr_time(), "$host:$share #$dumpNo" if ($opts{v}); |
|
my $db_user = $Conf{SearchUser} || ''; |
|
|
|
|
|
print STDERR curr_time(), "getting files for $host:$share increment $dumpNo..." if ($opts{v}); |
|
584 |
my $sql = q{ |
my $sql = q{ |
585 |
SELECT path,size |
SELECT path,size |
586 |
FROM files |
FROM files |
589 |
WHERE hosts.name = ? and shares.name = ? and backupnum = ? |
WHERE hosts.name = ? and shares.name = ? and backupnum = ? |
590 |
}; |
}; |
591 |
|
|
|
my $dbh = DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1} ); |
|
592 |
my $sth = $dbh->prepare($sql); |
my $sth = $dbh->prepare($sql); |
593 |
$sth->execute($host, $share, $dumpNo); |
$sth->execute($host, $share, $dumpNo); |
594 |
my $count = $sth->rows; |
my $count = $sth->rows; |
595 |
print STDERR " found $count items\n" if ($opts{v}); |
print STDERR " $count items, parts:" if ($opts{v}); |
596 |
while (my $row = $sth->fetchrow_arrayref) { |
while (my $row = $sth->fetchrow_arrayref) { |
597 |
#print STDERR "+ ", $row->[0],"\n"; |
#print STDERR "+ ", $row->[0],"\n"; |
598 |
$in_backup_increment->{ $row->[0] } = $row->[1]; |
$in_backup_increment->{ $row->[0] } = $row->[1]; |
599 |
} |
} |
600 |
|
|
601 |
$sth->finish(); |
$sth->finish(); |
|
$dbh->disconnect(); |
|
602 |
|
|
603 |
return $count; |
return $count; |
604 |
} |
} |
640 |
# is this file too large to fit into MaxArchiveFileSize? |
# is this file too large to fit into MaxArchiveFileSize? |
641 |
|
|
642 |
if ( ($current_tar_size + tar_overhead($tarPath) + $size) > $max_file_size ) { |
if ( ($current_tar_size + tar_overhead($tarPath) + $size) > $max_file_size ) { |
643 |
print STDERR "# tar file $current_tar_size + $tar_header_length + $size > $max_file_size, splitting\n" if ($opts{d}); |
print STDERR "# tar file $current_tar_size + $tar_header_length + $size > $max_file_size, splitting\n" if ($opts{d}); |
644 |
new_tar_part(); |
new_tar_part(); |
645 |
} |
} |
646 |
|
|
647 |
print STDERR "A $tarPath [$size] tell: $current_tar_size\n" if ($opts{d}); |
#print STDERR "A $tarPath [$size] tell: $current_tar_size\n" if ($opts{d}); |
648 |
|
$items_in_part++; |
649 |
|
|
650 |
if ( defined($PathRemove) |
if ( defined($PathRemove) |
651 |
&& substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) { |
&& substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) { |
718 |
TarWrite($fh, \$data); |
TarWrite($fh, \$data); |
719 |
$size += length($data); |
$size += length($data); |
720 |
TarWritePad($fh, $size); |
TarWritePad($fh, $size); |
721 |
|
|
722 |
|
$items_in_part++; |
723 |
} |
} |
724 |
$f->close; |
$f->close; |
725 |
$FileCnt++; |
$FileCnt++; |