| 1 |
#============================================================= -*-perl-*- |
| 2 |
# |
| 3 |
# Rsync package |
| 4 |
# |
| 5 |
# DESCRIPTION |
| 6 |
# |
| 7 |
# AUTHOR |
| 8 |
# Craig Barratt <cbarratt@users.sourceforge.net> |
| 9 |
# |
| 10 |
# COPYRIGHT |
| 11 |
# Copyright (C) 2002-2003 Craig Barratt |
| 12 |
# |
| 13 |
#======================================================================== |
| 14 |
# |
| 15 |
# Version 2.1.0, released 20 Jun 2004. |
| 16 |
# |
| 17 |
# See http://backuppc.sourceforge.net. |
| 18 |
# |
| 19 |
#======================================================================== |
| 20 |
|
| 21 |
package BackupPC::Xfer::RsyncFileIO; |
| 22 |
|
| 23 |
use strict; |
| 24 |
use File::Path; |
| 25 |
use BackupPC::Attrib qw(:all); |
| 26 |
use BackupPC::View; |
| 27 |
use BackupPC::Xfer::RsyncDigest qw(:all); |
| 28 |
use BackupPC::PoolWrite; |
| 29 |
|
| 30 |
use constant S_IFMT => 0170000; # type of file |
| 31 |
use constant S_IFDIR => 0040000; # directory |
| 32 |
use constant S_IFCHR => 0020000; # character special |
| 33 |
use constant S_IFBLK => 0060000; # block special |
| 34 |
use constant S_IFREG => 0100000; # regular |
| 35 |
use constant S_IFLNK => 0120000; # symbolic link |
| 36 |
use constant S_IFSOCK => 0140000; # socket |
| 37 |
use constant S_IFIFO => 0010000; # fifo |
| 38 |
|
| 39 |
use vars qw( $RsyncLibOK ); |
| 40 |
|
| 41 |
BEGIN { |
| 42 |
eval "use File::RsyncP::Digest"; |
| 43 |
if ( $@ ) { |
| 44 |
# |
| 45 |
# Rsync module doesn't exist. |
| 46 |
# |
| 47 |
$RsyncLibOK = 0; |
| 48 |
} else { |
| 49 |
$RsyncLibOK = 1; |
| 50 |
} |
| 51 |
}; |
| 52 |
|
| 53 |
sub new |
| 54 |
{ |
| 55 |
my($class, $options) = @_; |
| 56 |
|
| 57 |
return if ( !$RsyncLibOK ); |
| 58 |
$options ||= {}; |
| 59 |
my $fio = bless { |
| 60 |
blockSize => 700, |
| 61 |
logLevel => 0, |
| 62 |
digest => File::RsyncP::Digest->new, |
| 63 |
checksumSeed => 0, |
| 64 |
attrib => {}, |
| 65 |
logHandler => \&logHandler, |
| 66 |
stats => { |
| 67 |
errorCnt => 0, |
| 68 |
TotalFileCnt => 0, |
| 69 |
TotalFileSize => 0, |
| 70 |
ExistFileCnt => 0, |
| 71 |
ExistFileSize => 0, |
| 72 |
ExistFileCompSize => 0, |
| 73 |
}, |
| 74 |
%$options, |
| 75 |
}, $class; |
| 76 |
|
| 77 |
$fio->{shareM} = $fio->{bpc}->fileNameEltMangle($fio->{share}); |
| 78 |
$fio->{outDir} = "$fio->{xfer}{outDir}/new/"; |
| 79 |
$fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/"; |
| 80 |
$fio->{view} = BackupPC::View->new($fio->{bpc}, $fio->{client}, |
| 81 |
$fio->{backups}); |
| 82 |
$fio->{full} = $fio->{xfer}{type} eq "full" ? 1 : 0; |
| 83 |
$fio->{newFilesFH} = $fio->{xfer}{newFilesFH}; |
| 84 |
$fio->{partialNum} = undef if ( !$fio->{full} ); |
| 85 |
return $fio; |
| 86 |
} |
| 87 |
|
| 88 |
sub blockSize |
| 89 |
{ |
| 90 |
my($fio, $value) = @_; |
| 91 |
|
| 92 |
$fio->{blockSize} = $value if ( defined($value) ); |
| 93 |
return $fio->{blockSize}; |
| 94 |
} |
| 95 |
|
| 96 |
sub logHandlerSet |
| 97 |
{ |
| 98 |
my($fio, $sub) = @_; |
| 99 |
$fio->{logHandler} = $sub; |
| 100 |
BackupPC::Xfer::RsyncDigest->logHandlerSet($sub); |
| 101 |
} |
| 102 |
|
| 103 |
# |
| 104 |
# Setup rsync checksum computation for the given file. |
| 105 |
# |
| 106 |
sub csumStart |
| 107 |
{ |
| 108 |
my($fio, $f, $needMD4, $defBlkSize, $phase) = @_; |
| 109 |
|
| 110 |
$defBlkSize ||= $fio->{blockSize}; |
| 111 |
my $attr = $fio->attribGet($f); |
| 112 |
$fio->{file} = $f; |
| 113 |
$fio->csumEnd if ( defined($fio->{csum}) ); |
| 114 |
return -1 if ( $attr->{type} != BPC_FTYPE_FILE ); |
| 115 |
# |
| 116 |
# Rsync uses short checksums on the first phase. If the whole-file |
| 117 |
# checksum fails, then the file is repeated with full checksums. |
| 118 |
# So on phase 2 we verify the checksums if they are cached. |
| 119 |
# |
| 120 |
if ( ($phase > 0 || rand(1) < $fio->{cacheCheckProb}) |
| 121 |
&& $attr->{compress} |
| 122 |
&& $fio->{checksumSeed} == RSYNC_CSUMSEED_CACHE ) { |
| 123 |
my($err, $d, $blkSize) = BackupPC::Xfer::RsyncDigest->digestStart( |
| 124 |
$attr->{fullPath}, $attr->{size}, 0, |
| 125 |
$defBlkSize, $fio->{checksumSeed}, |
| 126 |
0, $attr->{compress}, 0); |
| 127 |
my($isCached, $isInvalid) = $d->isCached; |
| 128 |
if ( $fio->{logLevel} >= 5 ) { |
| 129 |
$fio->log("$attr->{fullPath} verify; cached = $isCached," |
| 130 |
. " invalid = $isInvalid, phase = $phase"); |
| 131 |
} |
| 132 |
if ( $isCached || $isInvalid ) { |
| 133 |
my $ret = BackupPC::Xfer::RsyncDigest->digestAdd( |
| 134 |
$attr->{fullPath}, $blkSize, |
| 135 |
$fio->{checksumSeed}, 1 # verify |
| 136 |
); |
| 137 |
if ( $ret != 1 ) { |
| 138 |
$fio->log("Bad cached digest for $attr->{fullPath} ($ret);" |
| 139 |
. " fixed"); |
| 140 |
$fio->{stats}{errorCnt}++; |
| 141 |
} else { |
| 142 |
$fio->log("$f->{name}: verified cached digest") |
| 143 |
if ( $fio->{logLevel} >= 2 ); |
| 144 |
} |
| 145 |
} |
| 146 |
$d->digestEnd; |
| 147 |
} |
| 148 |
(my $err, $fio->{csum}, my $blkSize) |
| 149 |
= BackupPC::Xfer::RsyncDigest->digestStart($attr->{fullPath}, |
| 150 |
$attr->{size}, 0, $defBlkSize, $fio->{checksumSeed}, |
| 151 |
$needMD4, $attr->{compress}, 1); |
| 152 |
if ( $fio->{logLevel} >= 5 ) { |
| 153 |
my($isCached, $invalid) = $fio->{csum}->isCached; |
| 154 |
$fio->log("$attr->{fullPath} cache = $isCached," |
| 155 |
. " invalid = $invalid, phase = $phase"); |
| 156 |
} |
| 157 |
if ( $err ) { |
| 158 |
$fio->log("Can't get rsync digests from $attr->{fullPath}" |
| 159 |
. " (err=$err, name=$f->{name})"); |
| 160 |
$fio->{stats}{errorCnt}++; |
| 161 |
return -1; |
| 162 |
} |
| 163 |
return $blkSize; |
| 164 |
} |
| 165 |
|
| 166 |
sub csumGet |
| 167 |
{ |
| 168 |
my($fio, $num, $csumLen, $blockSize) = @_; |
| 169 |
my($fileData); |
| 170 |
|
| 171 |
$num ||= 100; |
| 172 |
$csumLen ||= 16; |
| 173 |
return if ( !defined($fio->{csum}) ); |
| 174 |
return $fio->{csum}->digestGet($num, $csumLen); |
| 175 |
} |
| 176 |
|
| 177 |
sub csumEnd |
| 178 |
{ |
| 179 |
my($fio) = @_; |
| 180 |
|
| 181 |
return if ( !defined($fio->{csum}) ); |
| 182 |
return $fio->{csum}->digestEnd(); |
| 183 |
} |
| 184 |
|
| 185 |
sub readStart |
| 186 |
{ |
| 187 |
my($fio, $f) = @_; |
| 188 |
|
| 189 |
my $attr = $fio->attribGet($f); |
| 190 |
$fio->{file} = $f; |
| 191 |
$fio->readEnd if ( defined($fio->{fh}) ); |
| 192 |
if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath}, |
| 193 |
0, |
| 194 |
$attr->{compress})) ) { |
| 195 |
$fio->log("Can't open $attr->{fullPath} (name=$f->{name})"); |
| 196 |
$fio->{stats}{errorCnt}++; |
| 197 |
return; |
| 198 |
} |
| 199 |
$fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 ); |
| 200 |
} |
| 201 |
|
| 202 |
sub read |
| 203 |
{ |
| 204 |
my($fio, $num) = @_; |
| 205 |
my $fileData; |
| 206 |
|
| 207 |
$num ||= 32768; |
| 208 |
return if ( !defined($fio->{fh}) ); |
| 209 |
if ( $fio->{fh}->read(\$fileData, $num) <= 0 ) { |
| 210 |
return $fio->readEnd; |
| 211 |
} |
| 212 |
$fio->log(sprintf("read returns %d bytes", length($fileData))) |
| 213 |
if ( $fio->{logLevel} >= 8 ); |
| 214 |
return \$fileData; |
| 215 |
} |
| 216 |
|
| 217 |
sub readEnd |
| 218 |
{ |
| 219 |
my($fio) = @_; |
| 220 |
|
| 221 |
return if ( !defined($fio->{fh}) ); |
| 222 |
$fio->{fh}->close; |
| 223 |
$fio->log("closing $fio->{file}{name})") if ( $fio->{logLevel} >= 8 ); |
| 224 |
delete($fio->{fh}); |
| 225 |
return; |
| 226 |
} |
| 227 |
|
| 228 |
sub checksumSeed |
| 229 |
{ |
| 230 |
my($fio, $checksumSeed) = @_; |
| 231 |
|
| 232 |
$fio->{checksumSeed} = $checksumSeed; |
| 233 |
$fio->log("Checksum caching enabled (checksumSeed = $checksumSeed)") |
| 234 |
if ( $fio->{logLevel} >= 1 && $checksumSeed == RSYNC_CSUMSEED_CACHE ); |
| 235 |
$fio->log("Checksum seed is $checksumSeed") |
| 236 |
if ( $fio->{logLevel} >= 2 && $checksumSeed != RSYNC_CSUMSEED_CACHE ); |
| 237 |
} |
| 238 |
|
| 239 |
sub dirs |
| 240 |
{ |
| 241 |
my($fio, $localDir, $remoteDir) = @_; |
| 242 |
|
| 243 |
$fio->{localDir} = $localDir; |
| 244 |
$fio->{remoteDir} = $remoteDir; |
| 245 |
} |
| 246 |
|
| 247 |
sub viewCacheDir |
| 248 |
{ |
| 249 |
my($fio, $share, $dir) = @_; |
| 250 |
my $shareM; |
| 251 |
|
| 252 |
#$fio->log("viewCacheDir($share, $dir)"); |
| 253 |
if ( !defined($share) ) { |
| 254 |
$share = $fio->{share}; |
| 255 |
$shareM = $fio->{shareM}; |
| 256 |
} else { |
| 257 |
$shareM = $fio->{bpc}->fileNameEltMangle($share); |
| 258 |
} |
| 259 |
$shareM = "$shareM/$dir" if ( $dir ne "" ); |
| 260 |
return if ( defined($fio->{viewCache}{$shareM}) ); |
| 261 |
# |
| 262 |
# purge old cache entries (ie: those that don't match the |
| 263 |
# first part of $dir). |
| 264 |
# |
| 265 |
foreach my $d ( keys(%{$fio->{viewCache}}) ) { |
| 266 |
delete($fio->{viewCache}{$d}) if ( $shareM !~ m{^\Q$d/} ); |
| 267 |
} |
| 268 |
# |
| 269 |
# fetch new directory attributes |
| 270 |
# |
| 271 |
$fio->{viewCache}{$shareM} |
| 272 |
= $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir); |
| 273 |
# |
| 274 |
# also cache partial backup attrib data too |
| 275 |
# |
| 276 |
if ( defined($fio->{partialNum}) ) { |
| 277 |
foreach my $d ( keys(%{$fio->{partialCache}}) ) { |
| 278 |
delete($fio->{partialCache}{$d}) if ( $shareM !~ m{^\Q$d/} ); |
| 279 |
} |
| 280 |
$fio->{partialCache}{$shareM} |
| 281 |
= $fio->{view}->dirAttrib($fio->{partialNum}, $share, $dir); |
| 282 |
} |
| 283 |
} |
| 284 |
|
| 285 |
sub attribGetWhere |
| 286 |
{ |
| 287 |
my($fio, $f) = @_; |
| 288 |
my($dir, $fname, $share, $shareM); |
| 289 |
|
| 290 |
$fname = $f->{name}; |
| 291 |
$fname = "$fio->{xfer}{pathHdrSrc}/$fname" |
| 292 |
if ( defined($fio->{xfer}{pathHdrSrc}) ); |
| 293 |
$fname =~ s{//+}{/}g; |
| 294 |
if ( $fname =~ m{(.*)/(.*)} ) { |
| 295 |
$shareM = $fio->{shareM}; |
| 296 |
$dir = $1; |
| 297 |
$fname = $2; |
| 298 |
} elsif ( $fname ne "." ) { |
| 299 |
$shareM = $fio->{shareM}; |
| 300 |
$dir = ""; |
| 301 |
} else { |
| 302 |
$share = ""; |
| 303 |
$shareM = ""; |
| 304 |
$dir = ""; |
| 305 |
$fname = $fio->{share}; |
| 306 |
} |
| 307 |
$fio->viewCacheDir($share, $dir); |
| 308 |
$shareM .= "/$dir" if ( $dir ne "" ); |
| 309 |
if ( defined(my $attr = $fio->{viewCache}{$shareM}{$fname}) ) { |
| 310 |
return ($attr, 0); |
| 311 |
} elsif ( defined(my $attr = $fio->{partialCache}{$shareM}{$fname}) ) { |
| 312 |
return ($attr, 1); |
| 313 |
} else { |
| 314 |
return; |
| 315 |
} |
| 316 |
} |
| 317 |
|
| 318 |
sub attribGet |
| 319 |
{ |
| 320 |
my($fio, $f) = @_; |
| 321 |
|
| 322 |
my($attr) = $fio->attribGetWhere($f); |
| 323 |
return $attr; |
| 324 |
} |
| 325 |
|
| 326 |
sub mode2type |
| 327 |
{ |
| 328 |
my($fio, $mode) = @_; |
| 329 |
|
| 330 |
if ( ($mode & S_IFMT) == S_IFREG ) { |
| 331 |
return BPC_FTYPE_FILE; |
| 332 |
} elsif ( ($mode & S_IFMT) == S_IFDIR ) { |
| 333 |
return BPC_FTYPE_DIR; |
| 334 |
} elsif ( ($mode & S_IFMT) == S_IFLNK ) { |
| 335 |
return BPC_FTYPE_SYMLINK; |
| 336 |
} elsif ( ($mode & S_IFMT) == S_IFCHR ) { |
| 337 |
return BPC_FTYPE_CHARDEV; |
| 338 |
} elsif ( ($mode & S_IFMT) == S_IFBLK ) { |
| 339 |
return BPC_FTYPE_BLOCKDEV; |
| 340 |
} elsif ( ($mode & S_IFMT) == S_IFIFO ) { |
| 341 |
return BPC_FTYPE_FIFO; |
| 342 |
} elsif ( ($mode & S_IFMT) == S_IFSOCK ) { |
| 343 |
return BPC_FTYPE_SOCKET; |
| 344 |
} else { |
| 345 |
return BPC_FTYPE_UNKNOWN; |
| 346 |
} |
| 347 |
} |
| 348 |
|
| 349 |
# |
| 350 |
# Set the attributes for a file. Returns non-zero on error. |
| 351 |
# |
| 352 |
sub attribSet |
| 353 |
{ |
| 354 |
my($fio, $f, $placeHolder) = @_; |
| 355 |
my($dir, $file); |
| 356 |
|
| 357 |
if ( $f->{name} =~ m{(.*)/(.*)} ) { |
| 358 |
$file = $2; |
| 359 |
$dir = "$fio->{shareM}/" . $1; |
| 360 |
} elsif ( $f->{name} eq "." ) { |
| 361 |
$dir = ""; |
| 362 |
$file = $fio->{share}; |
| 363 |
} else { |
| 364 |
$dir = $fio->{shareM}; |
| 365 |
$file = $f->{name}; |
| 366 |
} |
| 367 |
|
| 368 |
if ( !defined($fio->{attribLastDir}) || $fio->{attribLastDir} ne $dir ) { |
| 369 |
# |
| 370 |
# Flush any directories that don't match the first part |
| 371 |
# of the new directory |
| 372 |
# |
| 373 |
foreach my $d ( keys(%{$fio->{attrib}}) ) { |
| 374 |
next if ( $d eq "" || "$dir/" =~ m{^\Q$d/} ); |
| 375 |
$fio->attribWrite($d); |
| 376 |
} |
| 377 |
$fio->{attribLastDir} = $dir; |
| 378 |
} |
| 379 |
if ( !exists($fio->{attrib}{$dir}) ) { |
| 380 |
$fio->{attrib}{$dir} = BackupPC::Attrib->new({ |
| 381 |
compress => $fio->{xfer}{compress}, |
| 382 |
}); |
| 383 |
my $path = $fio->{outDir} . $dir; |
| 384 |
if ( -f $fio->{attrib}{$dir}->fileName($path) |
| 385 |
&& !$fio->{attrib}{$dir}->read($path) ) { |
| 386 |
$fio->log(sprintf("Unable to read attribute file %s", |
| 387 |
$fio->{attrib}{$dir}->fileName($path))); |
| 388 |
} |
| 389 |
} |
| 390 |
$fio->log("attribSet(dir=$dir, file=$file)") if ( $fio->{logLevel} >= 4 ); |
| 391 |
|
| 392 |
$fio->{attrib}{$dir}->set($file, { |
| 393 |
type => $fio->mode2type($f->{mode}), |
| 394 |
mode => $f->{mode}, |
| 395 |
uid => $f->{uid}, |
| 396 |
gid => $f->{gid}, |
| 397 |
size => $placeHolder ? -1 : $f->{size}, |
| 398 |
mtime => $f->{mtime}, |
| 399 |
}); |
| 400 |
return; |
| 401 |
} |
| 402 |
|
| 403 |
sub attribWrite |
| 404 |
{ |
| 405 |
my($fio, $d) = @_; |
| 406 |
my($poolWrite); |
| 407 |
|
| 408 |
if ( !defined($d) ) { |
| 409 |
# |
| 410 |
# flush all entries (in reverse order) |
| 411 |
# |
| 412 |
foreach $d ( sort({$b cmp $a} keys(%{$fio->{attrib}})) ) { |
| 413 |
$fio->attribWrite($d); |
| 414 |
} |
| 415 |
return; |
| 416 |
} |
| 417 |
return if ( !defined($fio->{attrib}{$d}) ); |
| 418 |
# |
| 419 |
# Set deleted files in the attributes. Any file in the view |
| 420 |
# that doesn't have attributes is flagged as deleted for |
| 421 |
# incremental dumps. All files sent by rsync have attributes |
| 422 |
# temporarily set so we can do deletion detection. We also |
| 423 |
# prune these temporary attributes. |
| 424 |
# |
| 425 |
if ( $d ne "" ) { |
| 426 |
my $dir; |
| 427 |
my $share; |
| 428 |
|
| 429 |
$dir = $1 if ( $d =~ m{.+?/(.*)} ); |
| 430 |
$fio->viewCacheDir(undef, $dir); |
| 431 |
##print("attribWrite $d,$dir\n"); |
| 432 |
##$Data::Dumper::Indent = 1; |
| 433 |
##$fio->log("attribWrite $d,$dir"); |
| 434 |
##$fio->log("viewCacheLogKeys = ", keys(%{$fio->{viewCache}})); |
| 435 |
##$fio->log("attribKeys = ", keys(%{$fio->{attrib}})); |
| 436 |
##print "viewCache = ", Dumper($fio->{attrib}); |
| 437 |
##print "attrib = ", Dumper($fio->{attrib}); |
| 438 |
if ( defined($fio->{viewCache}{$d}) ) { |
| 439 |
foreach my $f ( keys(%{$fio->{viewCache}{$d}}) ) { |
| 440 |
my $name = $f; |
| 441 |
$name = "$1/$name" if ( $d =~ m{.*?/(.*)} ); |
| 442 |
if ( defined(my $a = $fio->{attrib}{$d}->get($f)) ) { |
| 443 |
# |
| 444 |
# delete temporary attributes (skipped files) |
| 445 |
# |
| 446 |
if ( $a->{size} < 0 ) { |
| 447 |
$fio->{attrib}{$d}->set($f, undef); |
| 448 |
$fio->logFileAction("skip", { |
| 449 |
%{$fio->{viewCache}{$d}{$f}}, |
| 450 |
name => $name, |
| 451 |
}) if ( $fio->{logLevel} >= 2 ); |
| 452 |
} |
| 453 |
} elsif ( !$fio->{full} ) { |
| 454 |
##print("Delete file $f\n"); |
| 455 |
$fio->logFileAction("delete", { |
| 456 |
%{$fio->{viewCache}{$d}{$f}}, |
| 457 |
name => $name, |
| 458 |
}) if ( $fio->{logLevel} >= 1 ); |
| 459 |
$fio->{attrib}{$d}->set($f, { |
| 460 |
type => BPC_FTYPE_DELETED, |
| 461 |
mode => 0, |
| 462 |
uid => 0, |
| 463 |
gid => 0, |
| 464 |
size => 0, |
| 465 |
mtime => 0, |
| 466 |
}); |
| 467 |
} |
| 468 |
} |
| 469 |
} |
| 470 |
} |
| 471 |
if ( $fio->{attrib}{$d}->fileCount ) { |
| 472 |
my $data = $fio->{attrib}{$d}->writeData; |
| 473 |
my $dirM = $d; |
| 474 |
|
| 475 |
$dirM = $1 . "/" . $fio->{bpc}->fileNameMangle($2) |
| 476 |
if ( $dirM =~ m{(.*?)/(.*)} ); |
| 477 |
my $fileName = $fio->{attrib}{$d}->fileName("$fio->{outDir}$dirM"); |
| 478 |
$fio->log("attribWrite(dir=$d) -> $fileName") |
| 479 |
if ( $fio->{logLevel} >= 4 ); |
| 480 |
my $poolWrite = BackupPC::PoolWrite->new($fio->{bpc}, $fileName, |
| 481 |
length($data), $fio->{xfer}{compress}); |
| 482 |
$poolWrite->write(\$data); |
| 483 |
$fio->processClose($poolWrite, $fio->{attrib}{$d}->fileName($dirM), |
| 484 |
length($data), 0); |
| 485 |
} |
| 486 |
delete($fio->{attrib}{$d}); |
| 487 |
} |
| 488 |
|
| 489 |
sub processClose |
| 490 |
{ |
| 491 |
my($fio, $poolWrite, $fileName, $origSize, $doStats) = @_; |
| 492 |
my($exists, $digest, $outSize, $errs) = $poolWrite->close; |
| 493 |
|
| 494 |
$fileName =~ s{^/+}{}; |
| 495 |
$fio->log(@$errs) if ( defined($errs) && @$errs ); |
| 496 |
if ( $doStats ) { |
| 497 |
$fio->{stats}{TotalFileCnt}++; |
| 498 |
$fio->{stats}{TotalFileSize} += $origSize; |
| 499 |
} |
| 500 |
if ( $exists ) { |
| 501 |
if ( $doStats ) { |
| 502 |
$fio->{stats}{ExistFileCnt}++; |
| 503 |
$fio->{stats}{ExistFileSize} += $origSize; |
| 504 |
$fio->{stats}{ExistFileCompSize} += $outSize; |
| 505 |
} |
| 506 |
} elsif ( $outSize > 0 ) { |
| 507 |
my $fh = $fio->{newFilesFH}; |
| 508 |
print($fh "$digest $origSize $fileName\n") if ( defined($fh) ); |
| 509 |
} |
| 510 |
return $exists && $origSize > 0; |
| 511 |
} |
| 512 |
|
| 513 |
sub statsGet |
| 514 |
{ |
| 515 |
my($fio) = @_; |
| 516 |
|
| 517 |
return $fio->{stats}; |
| 518 |
} |
| 519 |
|
| 520 |
# |
| 521 |
# Make a given directory. Returns non-zero on error. |
| 522 |
# |
| 523 |
sub makePath |
| 524 |
{ |
| 525 |
my($fio, $f) = @_; |
| 526 |
my $name = $1 if ( $f->{name} =~ /(.*)/ ); |
| 527 |
my $path; |
| 528 |
|
| 529 |
if ( $name eq "." ) { |
| 530 |
$path = $fio->{outDirSh}; |
| 531 |
} else { |
| 532 |
$path = $fio->{outDirSh} . $fio->{bpc}->fileNameMangle($name); |
| 533 |
} |
| 534 |
$fio->logFileAction("create", $f) if ( $fio->{logLevel} >= 1 ); |
| 535 |
$fio->log("makePath($path, 0777)") if ( $fio->{logLevel} >= 5 ); |
| 536 |
$path = $1 if ( $path =~ /(.*)/ ); |
| 537 |
File::Path::mkpath($path, 0, 0777) if ( !-d $path ); |
| 538 |
return $fio->attribSet($f) if ( -d $path ); |
| 539 |
$fio->log("Can't create directory $path"); |
| 540 |
$fio->{stats}{errorCnt}++; |
| 541 |
return -1; |
| 542 |
} |
| 543 |
|
| 544 |
# |
| 545 |
# Make a special file. Returns non-zero on error. |
| 546 |
# |
| 547 |
sub makeSpecial |
| 548 |
{ |
| 549 |
my($fio, $f) = @_; |
| 550 |
my $name = $1 if ( $f->{name} =~ /(.*)/ ); |
| 551 |
my $fNameM = $fio->{bpc}->fileNameMangle($name); |
| 552 |
my $path = $fio->{outDirSh} . $fNameM; |
| 553 |
my $attr = $fio->attribGet($f); |
| 554 |
my $str = ""; |
| 555 |
my $type = $fio->mode2type($f->{mode}); |
| 556 |
|
| 557 |
$fio->log("makeSpecial($path, $type, $f->{mode})") |
| 558 |
if ( $fio->{logLevel} >= 5 ); |
| 559 |
if ( $type == BPC_FTYPE_CHARDEV || $type == BPC_FTYPE_BLOCKDEV ) { |
| 560 |
my($major, $minor, $fh, $fileData); |
| 561 |
|
| 562 |
$major = $f->{rdev} >> 8; |
| 563 |
$minor = $f->{rdev} & 0xff; |
| 564 |
$str = "$major,$minor"; |
| 565 |
} elsif ( ($f->{mode} & S_IFMT) == S_IFLNK ) { |
| 566 |
$str = $f->{link}; |
| 567 |
} |
| 568 |
# |
| 569 |
# Now see if the file is different, or this is a full, in which |
| 570 |
# case we create the new file. |
| 571 |
# |
| 572 |
my($fh, $fileData); |
| 573 |
if ( $fio->{full} |
| 574 |
|| !defined($attr) |
| 575 |
|| $attr->{type} != $fio->mode2type($f->{mode}) |
| 576 |
|| $attr->{mtime} != $f->{mtime} |
| 577 |
|| $attr->{size} != $f->{size} |
| 578 |
|| $attr->{uid} != $f->{uid} |
| 579 |
|| $attr->{gid} != $f->{gid} |
| 580 |
|| $attr->{mode} != $f->{mode} |
| 581 |
|| !defined($fh = BackupPC::FileZIO->open($attr->{fullPath}, 0, |
| 582 |
$attr->{compress})) |
| 583 |
|| $fh->read(\$fileData, length($str) + 1) != length($str) |
| 584 |
|| $fileData ne $str ) { |
| 585 |
$fh->close if ( defined($fh) ); |
| 586 |
$fh = BackupPC::PoolWrite->new($fio->{bpc}, $path, |
| 587 |
length($str), $fio->{xfer}{compress}); |
| 588 |
$fh->write(\$str); |
| 589 |
my $exist = $fio->processClose($fh, "$fio->{shareM}/$fNameM", |
| 590 |
length($str), 1); |
| 591 |
$fio->logFileAction($exist ? "pool" : "create", $f) |
| 592 |
if ( $fio->{logLevel} >= 1 ); |
| 593 |
return $fio->attribSet($f); |
| 594 |
} else { |
| 595 |
$fio->logFileAction("skip", $f) if ( $fio->{logLevel} >= 2 ); |
| 596 |
} |
| 597 |
$fh->close if ( defined($fh) ); |
| 598 |
} |
| 599 |
|
| 600 |
sub unlink |
| 601 |
{ |
| 602 |
my($fio, $path) = @_; |
| 603 |
|
| 604 |
$fio->log("Unexpected call BackupPC::Xfer::RsyncFileIO->unlink($path)"); |
| 605 |
} |
| 606 |
|
| 607 |
# |
| 608 |
# Default log handler |
| 609 |
# |
| 610 |
sub logHandler |
| 611 |
{ |
| 612 |
my($str) = @_; |
| 613 |
|
| 614 |
print(STDERR $str, "\n"); |
| 615 |
} |
| 616 |
|
| 617 |
# |
| 618 |
# Handle one or more log messages |
| 619 |
# |
| 620 |
sub log |
| 621 |
{ |
| 622 |
my($fio, @logStr) = @_; |
| 623 |
|
| 624 |
foreach my $str ( @logStr ) { |
| 625 |
next if ( $str eq "" ); |
| 626 |
$fio->{logHandler}($str); |
| 627 |
} |
| 628 |
} |
| 629 |
|
| 630 |
# |
| 631 |
# Generate a log file message for a completed file |
| 632 |
# |
| 633 |
sub logFileAction |
| 634 |
{ |
| 635 |
my($fio, $action, $f) = @_; |
| 636 |
my $owner = "$f->{uid}/$f->{gid}"; |
| 637 |
my $type = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s")) |
| 638 |
[($f->{mode} & S_IFMT) >> 12]; |
| 639 |
|
| 640 |
$fio->log(sprintf(" %-6s %1s%4o %9s %11.0f %s", |
| 641 |
$action, |
| 642 |
$type, |
| 643 |
$f->{mode} & 07777, |
| 644 |
$owner, |
| 645 |
$f->{size}, |
| 646 |
$f->{name})); |
| 647 |
} |
| 648 |
|
| 649 |
# |
| 650 |
# If there is a partial and we are doing a full, we do an incremental |
| 651 |
# against the partial and a full against the rest. This subroutine |
| 652 |
# is how we tell File::RsyncP which files to ignore attributes on |
| 653 |
# (ie: against the partial dump we do consider the attributes, but |
| 654 |
# otherwise we ignore attributes). |
| 655 |
# |
| 656 |
sub ignoreAttrOnFile |
| 657 |
{ |
| 658 |
my($fio, $f) = @_; |
| 659 |
|
| 660 |
return if ( !defined($fio->{partialNum}) ); |
| 661 |
my($attr, $isPartial) = $fio->attribGetWhere($f); |
| 662 |
$fio->log("$f->{name}: just checking attributes from partial") |
| 663 |
if ( $isPartial && $fio->{logLevel} >= 5 ); |
| 664 |
return !$isPartial; |
| 665 |
} |
| 666 |
|
| 667 |
# |
| 668 |
# This is called by File::RsyncP when a file is skipped because the |
| 669 |
# attributes match. |
| 670 |
# |
| 671 |
sub attrSkippedFile |
| 672 |
{ |
| 673 |
my($fio, $f, $attr) = @_; |
| 674 |
|
| 675 |
# |
| 676 |
# Unless this is a partial, this is normal so ignore it. |
| 677 |
# |
| 678 |
return if ( !defined($fio->{partialNum}) ); |
| 679 |
|
| 680 |
$fio->log("$f->{name}: skipped in partial; adding link") |
| 681 |
if ( $fio->{logLevel} >= 5 ); |
| 682 |
$fio->{rxLocalAttr} = $attr; |
| 683 |
$fio->{rxFile} = $f; |
| 684 |
$fio->{rxSize} = $attr->{size}; |
| 685 |
delete($fio->{rxInFd}); |
| 686 |
delete($fio->{rxOutFd}); |
| 687 |
delete($fio->{rxDigest}); |
| 688 |
delete($fio->{rxInData}); |
| 689 |
return $fio->fileDeltaRxDone(); |
| 690 |
} |
| 691 |
|
| 692 |
# |
| 693 |
# Start receive of file deltas for a particular file. |
| 694 |
# |
| 695 |
sub fileDeltaRxStart |
| 696 |
{ |
| 697 |
my($fio, $f, $cnt, $size, $remainder) = @_; |
| 698 |
|
| 699 |
$fio->{rxFile} = $f; # remote file attributes |
| 700 |
$fio->{rxLocalAttr} = $fio->attribGet($f); # local file attributes |
| 701 |
$fio->{rxBlkCnt} = $cnt; # how many blocks we will receive |
| 702 |
$fio->{rxBlkSize} = $size; # block size |
| 703 |
$fio->{rxRemainder} = $remainder; # size of the last block |
| 704 |
$fio->{rxMatchBlk} = 0; # current start of match |
| 705 |
$fio->{rxMatchNext} = 0; # current next block of match |
| 706 |
$fio->{rxSize} = 0; # size of received file |
| 707 |
my $rxSize = $cnt > 0 ? ($cnt - 1) * $size + $remainder : 0; |
| 708 |
if ( $fio->{rxFile}{size} != $rxSize ) { |
| 709 |
$fio->{rxMatchBlk} = undef; # size different, so no file match |
| 710 |
$fio->log("$fio->{rxFile}{name}: size doesn't match" |
| 711 |
. " ($fio->{rxFile}{size} vs $rxSize)") |
| 712 |
if ( $fio->{logLevel} >= 5 ); |
| 713 |
} |
| 714 |
delete($fio->{rxInFd}); |
| 715 |
delete($fio->{rxOutFd}); |
| 716 |
delete($fio->{rxDigest}); |
| 717 |
delete($fio->{rxInData}); |
| 718 |
} |
| 719 |
|
| 720 |
# |
| 721 |
# Process the next file delta for the current file. Returns 0 if ok, |
| 722 |
# -1 if not. Must be called with either a block number, $blk, or new data, |
| 723 |
# $newData, (not both) defined. |
| 724 |
# |
| 725 |
sub fileDeltaRxNext |
| 726 |
{ |
| 727 |
my($fio, $blk, $newData) = @_; |
| 728 |
|
| 729 |
if ( defined($blk) ) { |
| 730 |
if ( defined($fio->{rxMatchBlk}) && $fio->{rxMatchNext} == $blk ) { |
| 731 |
# |
| 732 |
# got the next block in order; just keep track. |
| 733 |
# |
| 734 |
$fio->{rxMatchNext}++; |
| 735 |
return; |
| 736 |
} |
| 737 |
} |
| 738 |
my $newDataLen = length($newData); |
| 739 |
$fio->log("$fio->{rxFile}{name}: blk=$blk, newData=$newDataLen, rxMatchBlk=$fio->{rxMatchBlk}, rxMatchNext=$fio->{rxMatchNext}") |
| 740 |
if ( $fio->{logLevel} >= 8 ); |
| 741 |
if ( !defined($fio->{rxOutFd}) ) { |
| 742 |
# |
| 743 |
# maybe the file has no changes |
| 744 |
# |
| 745 |
if ( $fio->{rxMatchNext} == $fio->{rxBlkCnt} |
| 746 |
&& !defined($blk) && !defined($newData) ) { |
| 747 |
#$fio->log("$fio->{rxFile}{name}: file is unchanged"); |
| 748 |
# if ( $fio->{logLevel} >= 8 ); |
| 749 |
return; |
| 750 |
} |
| 751 |
|
| 752 |
# |
| 753 |
# need to open an output file where we will build the |
| 754 |
# new version. |
| 755 |
# |
| 756 |
$fio->{rxFile}{name} =~ /(.*)/; |
| 757 |
my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($1); |
| 758 |
my $rxOutFile = $fio->{outDir} . $rxOutFileRel; |
| 759 |
$fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc}, |
| 760 |
$rxOutFile, $fio->{rxFile}{size}, |
| 761 |
$fio->{xfer}{compress}); |
| 762 |
$fio->log("$fio->{rxFile}{name}: opening output file $rxOutFile") |
| 763 |
if ( $fio->{logLevel} >= 9 ); |
| 764 |
$fio->{rxOutFile} = $rxOutFile; |
| 765 |
$fio->{rxOutFileRel} = $rxOutFileRel; |
| 766 |
$fio->{rxDigest} = File::RsyncP::Digest->new; |
| 767 |
$fio->{rxDigest}->add(pack("V", $fio->{checksumSeed})); |
| 768 |
} |
| 769 |
if ( defined($fio->{rxMatchBlk}) |
| 770 |
&& $fio->{rxMatchBlk} != $fio->{rxMatchNext} ) { |
| 771 |
# |
| 772 |
# Need to copy the sequence of blocks that matched. If the file |
| 773 |
# is compressed we need to make a copy of the uncompressed file, |
| 774 |
# since the compressed file is not seekable. Future optimizations |
| 775 |
# could include only creating an uncompressed copy if the matching |
| 776 |
# blocks were not monotonic, and to only do this if there are |
| 777 |
# matching blocks (eg, maybe the entire file is new). |
| 778 |
# |
| 779 |
my $attr = $fio->{rxLocalAttr}; |
| 780 |
my $fh; |
| 781 |
if ( !defined($fio->{rxInFd}) && !defined($fio->{rxInData}) ) { |
| 782 |
if ( $attr->{compress} ) { |
| 783 |
if ( !defined($fh = BackupPC::FileZIO->open( |
| 784 |
$attr->{fullPath}, |
| 785 |
0, |
| 786 |
$attr->{compress})) ) { |
| 787 |
$fio->log("Can't open $attr->{fullPath}"); |
| 788 |
$fio->{stats}{errorCnt}++; |
| 789 |
return -1; |
| 790 |
} |
| 791 |
if ( $attr->{size} < 16 * 1024 * 1024 ) { |
| 792 |
# |
| 793 |
# Cache the entire old file if it is less than 16MB |
| 794 |
# |
| 795 |
my $data; |
| 796 |
$fio->{rxInData} = ""; |
| 797 |
while ( $fh->read(\$data, 16 * 1024 * 1024) > 0 ) { |
| 798 |
$fio->{rxInData} .= $data; |
| 799 |
} |
| 800 |
$fio->log("$attr->{fullPath}: cached all $attr->{size}" |
| 801 |
. " bytes") |
| 802 |
if ( $fio->{logLevel} >= 9 ); |
| 803 |
} else { |
| 804 |
# |
| 805 |
# Create and write a temporary output file |
| 806 |
# |
| 807 |
unlink("$fio->{outDirSh}RStmp") |
| 808 |
if ( -f "$fio->{outDirSh}RStmp" ); |
| 809 |
if ( open(F, "+>", "$fio->{outDirSh}RStmp") ) { |
| 810 |
my $data; |
| 811 |
my $byteCnt = 0; |
| 812 |
binmode(F); |
| 813 |
while ( $fh->read(\$data, 1024 * 1024) > 0 ) { |
| 814 |
if ( syswrite(F, $data) != length($data) ) { |
| 815 |
$fio->log(sprintf("Can't write len=%d to %s", |
| 816 |
length($data) , "$fio->{outDirSh}RStmp")); |
| 817 |
$fh->close; |
| 818 |
$fio->{stats}{errorCnt}++; |
| 819 |
return -1; |
| 820 |
} |
| 821 |
$byteCnt += length($data); |
| 822 |
} |
| 823 |
$fio->{rxInFd} = *F; |
| 824 |
$fio->{rxInName} = "$fio->{outDirSh}RStmp"; |
| 825 |
sysseek($fio->{rxInFd}, 0, 0); |
| 826 |
$fio->log("$attr->{fullPath}: copied $byteCnt," |
| 827 |
. "$attr->{size} bytes to $fio->{rxInName}") |
| 828 |
if ( $fio->{logLevel} >= 9 ); |
| 829 |
} else { |
| 830 |
$fio->log("Unable to open $fio->{outDirSh}RStmp"); |
| 831 |
$fh->close; |
| 832 |
$fio->{stats}{errorCnt}++; |
| 833 |
return -1; |
| 834 |
} |
| 835 |
} |
| 836 |
$fh->close; |
| 837 |
} else { |
| 838 |
if ( open(F, "<", $attr->{fullPath}) ) { |
| 839 |
binmode(F); |
| 840 |
$fio->{rxInFd} = *F; |
| 841 |
$fio->{rxInName} = $attr->{fullPath}; |
| 842 |
} else { |
| 843 |
$fio->log("Unable to open $attr->{fullPath}"); |
| 844 |
$fio->{stats}{errorCnt}++; |
| 845 |
return -1; |
| 846 |
} |
| 847 |
} |
| 848 |
} |
| 849 |
my $lastBlk = $fio->{rxMatchNext} - 1; |
| 850 |
$fio->log("$fio->{rxFile}{name}: writing blocks $fio->{rxMatchBlk}.." |
| 851 |
. "$lastBlk") |
| 852 |
if ( $fio->{logLevel} >= 9 ); |
| 853 |
my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize}; |
| 854 |
if ( defined($fio->{rxInFd}) |
| 855 |
&& !sysseek($fio->{rxInFd}, $seekPosn, 0) ) { |
| 856 |
$fio->log("Unable to seek $attr->{rxInName} to $seekPosn"); |
| 857 |
$fio->{stats}{errorCnt}++; |
| 858 |
return -1; |
| 859 |
} |
| 860 |
my $cnt = $fio->{rxMatchNext} - $fio->{rxMatchBlk}; |
| 861 |
my($thisCnt, $len, $data); |
| 862 |
for ( my $i = 0 ; $i < $cnt ; $i += $thisCnt ) { |
| 863 |
$thisCnt = $cnt - $i; |
| 864 |
$thisCnt = 512 if ( $thisCnt > 512 ); |
| 865 |
if ( $fio->{rxMatchBlk} + $i + $thisCnt == $fio->{rxBlkCnt} ) { |
| 866 |
$len = ($thisCnt - 1) * $fio->{rxBlkSize} + $fio->{rxRemainder}; |
| 867 |
} else { |
| 868 |
$len = $thisCnt * $fio->{rxBlkSize}; |
| 869 |
} |
| 870 |
if ( defined($fio->{rxInData}) ) { |
| 871 |
$data = substr($fio->{rxInData}, $seekPosn, $len); |
| 872 |
$seekPosn += $len; |
| 873 |
} else { |
| 874 |
my $got = sysread($fio->{rxInFd}, $data, $len); |
| 875 |
if ( $got != $len ) { |
| 876 |
my $inFileSize = -s $fio->{rxInName}; |
| 877 |
$fio->log("Unable to read $len bytes from $fio->{rxInName}" |
| 878 |
. " got=$got, seekPosn=$seekPosn" |
| 879 |
. " ($i,$thisCnt,$fio->{rxBlkCnt},$inFileSize" |
| 880 |
. ",$attr->{size})"); |
| 881 |
$fio->{stats}{errorCnt}++; |
| 882 |
return -1; |
| 883 |
} |
| 884 |
$seekPosn += $len; |
| 885 |
} |
| 886 |
$fio->{rxOutFd}->write(\$data); |
| 887 |
$fio->{rxDigest}->add($data); |
| 888 |
$fio->{rxSize} += length($data); |
| 889 |
} |
| 890 |
$fio->{rxMatchBlk} = undef; |
| 891 |
} |
| 892 |
if ( defined($blk) ) { |
| 893 |
# |
| 894 |
# Remember the new block number |
| 895 |
# |
| 896 |
$fio->{rxMatchBlk} = $blk; |
| 897 |
$fio->{rxMatchNext} = $blk + 1; |
| 898 |
} |
| 899 |
if ( defined($newData) ) { |
| 900 |
# |
| 901 |
# Write the new chunk |
| 902 |
# |
| 903 |
my $len = length($newData); |
| 904 |
$fio->log("$fio->{rxFile}{name}: writing $len bytes new data") |
| 905 |
if ( $fio->{logLevel} >= 9 ); |
| 906 |
$fio->{rxOutFd}->write(\$newData); |
| 907 |
$fio->{rxDigest}->add($newData); |
| 908 |
$fio->{rxSize} += length($newData); |
| 909 |
} |
| 910 |
} |
| 911 |
|
| 912 |
# |
| 913 |
# Finish up the current receive file. Returns undef if ok, -1 if not. |
| 914 |
# Returns 1 if the md4 digest doesn't match. |
| 915 |
# |
| 916 |
sub fileDeltaRxDone |
| 917 |
{ |
| 918 |
my($fio, $md4, $phase) = @_; |
| 919 |
my $name = $1 if ( $fio->{rxFile}{name} =~ /(.*)/ ); |
| 920 |
my $ret; |
| 921 |
|
| 922 |
close($fio->{rxInFd}) if ( defined($fio->{rxInFd}) ); |
| 923 |
unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" ); |
| 924 |
|
| 925 |
# |
| 926 |
# Check the final md4 digest |
| 927 |
# |
| 928 |
if ( defined($md4) ) { |
| 929 |
my $newDigest; |
| 930 |
if ( !defined($fio->{rxDigest}) ) { |
| 931 |
# |
| 932 |
# File was exact match, but we still need to verify the |
| 933 |
# MD4 checksum. Compute the md4 digest (or fetch the |
| 934 |
# cached one.) |
| 935 |
# |
| 936 |
if ( defined(my $attr = $fio->{rxLocalAttr}) ) { |
| 937 |
# |
| 938 |
# block size doesn't matter: we're only going to |
| 939 |
# fetch the md4 file digest, not the block digests. |
| 940 |
# |
| 941 |
my($err, $csum, $blkSize) |
| 942 |
= BackupPC::Xfer::RsyncDigest->digestStart( |
| 943 |
$attr->{fullPath}, $attr->{size}, |
| 944 |
0, 2048, $fio->{checksumSeed}, 1, |
| 945 |
$attr->{compress}, 1); |
| 946 |
if ( $err ) { |
| 947 |
$fio->log("Can't open $attr->{fullPath} for MD4" |
| 948 |
. " check (err=$err, $name)"); |
| 949 |
$fio->{stats}{errorCnt}++; |
| 950 |
} else { |
| 951 |
if ( $fio->{logLevel} >= 5 ) { |
| 952 |
my($isCached, $invalid) = $csum->isCached; |
| 953 |
$fio->log("MD4 $attr->{fullPath} cache = $isCached," |
| 954 |
. " invalid = $invalid"); |
| 955 |
} |
| 956 |
$newDigest = $csum->digestEnd; |
| 957 |
} |
| 958 |
$fio->{rxSize} = $attr->{size}; |
| 959 |
} else { |
| 960 |
# |
| 961 |
# Empty file; just create an empty file digest |
| 962 |
# |
| 963 |
$fio->{rxDigest} = File::RsyncP::Digest->new; |
| 964 |
$fio->{rxDigest}->add(pack("V", $fio->{checksumSeed})); |
| 965 |
$newDigest = $fio->{rxDigest}->digest; |
| 966 |
} |
| 967 |
$fio->log("$name got exact match") if ( $fio->{logLevel} >= 5 ); |
| 968 |
} else { |
| 969 |
$newDigest = $fio->{rxDigest}->digest; |
| 970 |
} |
| 971 |
if ( $fio->{logLevel} >= 3 ) { |
| 972 |
my $md4Str = unpack("H*", $md4); |
| 973 |
my $newStr = unpack("H*", $newDigest); |
| 974 |
$fio->log("$name got digests $md4Str vs $newStr") |
| 975 |
} |
| 976 |
if ( $md4 ne $newDigest ) { |
| 977 |
if ( $phase > 0 ) { |
| 978 |
$fio->log("$name: fatal error: md4 doesn't match on retry;" |
| 979 |
. " file removed"); |
| 980 |
} else { |
| 981 |
$fio->log("$name: md4 doesn't match: will retry in phase 1;" |
| 982 |
. " file removed"); |
| 983 |
} |
| 984 |
$fio->{stats}{errorCnt}++; |
| 985 |
if ( defined($fio->{rxOutFd}) ) { |
| 986 |
$fio->{rxOutFd}->close; |
| 987 |
unlink($fio->{rxOutFile}); |
| 988 |
} |
| 989 |
delete($fio->{rxFile}); |
| 990 |
delete($fio->{rxOutFile}); |
| 991 |
return 1; |
| 992 |
} |
| 993 |
} |
| 994 |
|
| 995 |
# |
| 996 |
# One special case is an empty file: if the file size is |
| 997 |
# zero we need to open the output file to create it. |
| 998 |
# |
| 999 |
if ( $fio->{rxSize} == 0 ) { |
| 1000 |
my $rxOutFileRel = "$fio->{shareM}/" |
| 1001 |
. $fio->{bpc}->fileNameMangle($name); |
| 1002 |
my $rxOutFile = $fio->{outDir} . $rxOutFileRel; |
| 1003 |
$fio->{rxOutFd} = BackupPC::PoolWrite->new($fio->{bpc}, |
| 1004 |
$rxOutFile, $fio->{rxSize}, |
| 1005 |
$fio->{xfer}{compress}); |
| 1006 |
} |
| 1007 |
if ( !defined($fio->{rxOutFd}) ) { |
| 1008 |
# |
| 1009 |
# No output file, meaning original was an exact match. |
| 1010 |
# |
| 1011 |
$fio->log("$name: nothing to do") |
| 1012 |
if ( $fio->{logLevel} >= 5 ); |
| 1013 |
my $attr = $fio->{rxLocalAttr}; |
| 1014 |
my $f = $fio->{rxFile}; |
| 1015 |
$fio->logFileAction("same", $f) if ( $fio->{logLevel} >= 1 ); |
| 1016 |
if ( $fio->{full} |
| 1017 |
|| $attr->{type} != $f->{type} |
| 1018 |
|| $attr->{mtime} != $f->{mtime} |
| 1019 |
|| $attr->{size} != $f->{size} |
| 1020 |
|| $attr->{gid} != $f->{gid} |
| 1021 |
|| $attr->{mode} != $f->{mode} ) { |
| 1022 |
# |
| 1023 |
# In the full case, or if the attributes are different, |
| 1024 |
# we need to make a link from the previous file and |
| 1025 |
# set the attributes. |
| 1026 |
# |
| 1027 |
my $rxOutFile = $fio->{outDirSh} |
| 1028 |
. $fio->{bpc}->fileNameMangle($name); |
| 1029 |
if ( !link($attr->{fullPath}, $rxOutFile) ) { |
| 1030 |
$fio->log("Unable to link $attr->{fullPath} to $rxOutFile"); |
| 1031 |
$fio->{stats}{errorCnt}++; |
| 1032 |
$ret = -1; |
| 1033 |
} else { |
| 1034 |
# |
| 1035 |
# Cumulate the stats |
| 1036 |
# |
| 1037 |
$fio->{stats}{TotalFileCnt}++; |
| 1038 |
$fio->{stats}{TotalFileSize} += $fio->{rxSize}; |
| 1039 |
$fio->{stats}{ExistFileCnt}++; |
| 1040 |
$fio->{stats}{ExistFileSize} += $fio->{rxSize}; |
| 1041 |
$fio->{stats}{ExistFileCompSize} += -s $rxOutFile; |
| 1042 |
$fio->{rxFile}{size} = $fio->{rxSize}; |
| 1043 |
$ret = $fio->attribSet($fio->{rxFile}); |
| 1044 |
} |
| 1045 |
} |
| 1046 |
} else { |
| 1047 |
my $exist = $fio->processClose($fio->{rxOutFd}, |
| 1048 |
$fio->{rxOutFileRel}, |
| 1049 |
$fio->{rxSize}, 1); |
| 1050 |
$fio->logFileAction($exist ? "pool" : "create", $fio->{rxFile}) |
| 1051 |
if ( $fio->{logLevel} >= 1 ); |
| 1052 |
$fio->{rxFile}{size} = $fio->{rxSize}; |
| 1053 |
$ret = $fio->attribSet($fio->{rxFile}); |
| 1054 |
} |
| 1055 |
delete($fio->{rxDigest}); |
| 1056 |
delete($fio->{rxInData}); |
| 1057 |
delete($fio->{rxFile}); |
| 1058 |
delete($fio->{rxOutFile}); |
| 1059 |
return $ret; |
| 1060 |
} |
| 1061 |
|
| 1062 |
# |
| 1063 |
# Callback function for BackupPC::View->find. Note the order of the |
| 1064 |
# first two arguments. |
| 1065 |
# |
| 1066 |
sub fileListEltSend |
| 1067 |
{ |
| 1068 |
my($a, $fio, $fList, $outputFunc) = @_; |
| 1069 |
my $name = $a->{relPath}; |
| 1070 |
my $n = $name; |
| 1071 |
my $type = $fio->mode2type($a->{mode}); |
| 1072 |
my $extraAttribs = {}; |
| 1073 |
|
| 1074 |
$n =~ s/^\Q$fio->{xfer}{pathHdrSrc}//; |
| 1075 |
$fio->log("Sending $name (remote=$n)") if ( $fio->{logLevel} >= 4 ); |
| 1076 |
if ( $type == BPC_FTYPE_CHARDEV |
| 1077 |
|| $type == BPC_FTYPE_BLOCKDEV |
| 1078 |
|| $type == BPC_FTYPE_SYMLINK ) { |
| 1079 |
my $fh = BackupPC::FileZIO->open($a->{fullPath}, 0, $a->{compress}); |
| 1080 |
my($str, $rdSize); |
| 1081 |
if ( defined($fh) ) { |
| 1082 |
$rdSize = $fh->read(\$str, $a->{size} + 1024); |
| 1083 |
if ( $type == BPC_FTYPE_SYMLINK ) { |
| 1084 |
# |
| 1085 |
# Reconstruct symbolic link |
| 1086 |
# |
| 1087 |
$extraAttribs = { link => $str }; |
| 1088 |
if ( $rdSize != $a->{size} ) { |
| 1089 |
# ERROR |
| 1090 |
$fio->log("$name: can't read exactly $a->{size} bytes"); |
| 1091 |
$fio->{stats}{errorCnt}++; |
| 1092 |
} |
| 1093 |
} elsif ( $str =~ /(\d*),(\d*)/ ) { |
| 1094 |
# |
| 1095 |
# Reconstruct char or block special major/minor device num |
| 1096 |
# |
| 1097 |
# Note: char/block devices have $a->{size} = 0, so we |
| 1098 |
# can't do an error check on $rdSize. |
| 1099 |
# |
| 1100 |
$extraAttribs = { rdev => $1 * 256 + $2 }; |
| 1101 |
} else { |
| 1102 |
$fio->log("$name: unexpected special file contents $str"); |
| 1103 |
$fio->{stats}{errorCnt}++; |
| 1104 |
} |
| 1105 |
$fh->close; |
| 1106 |
} else { |
| 1107 |
# ERROR |
| 1108 |
$fio->log("$name: can't open"); |
| 1109 |
$fio->{stats}{errorCnt}++; |
| 1110 |
} |
| 1111 |
} |
| 1112 |
my $f = { |
| 1113 |
name => $n, |
| 1114 |
#dev => 0, # later, when we support hardlinks |
| 1115 |
#inode => 0, # later, when we support hardlinks |
| 1116 |
mode => $a->{mode}, |
| 1117 |
uid => $a->{uid}, |
| 1118 |
gid => $a->{gid}, |
| 1119 |
mtime => $a->{mtime}, |
| 1120 |
size => $a->{size}, |
| 1121 |
%$extraAttribs, |
| 1122 |
}; |
| 1123 |
$fList->encode($f); |
| 1124 |
$f->{name} = "$fio->{xfer}{pathHdrDest}/$f->{name}"; |
| 1125 |
$f->{name} =~ s{//+}{/}g; |
| 1126 |
$fio->logFileAction("restore", $f) if ( $fio->{logLevel} >= 1 ); |
| 1127 |
&$outputFunc($fList->encodeData); |
| 1128 |
# |
| 1129 |
# Cumulate stats |
| 1130 |
# |
| 1131 |
if ( $type != BPC_FTYPE_DIR ) { |
| 1132 |
$fio->{stats}{TotalFileCnt}++; |
| 1133 |
$fio->{stats}{TotalFileSize} += $a->{size}; |
| 1134 |
} |
| 1135 |
} |
| 1136 |
|
| 1137 |
sub fileListSend |
| 1138 |
{ |
| 1139 |
my($fio, $flist, $outputFunc) = @_; |
| 1140 |
|
| 1141 |
# |
| 1142 |
# Populate the file list with the files requested by the user. |
| 1143 |
# Since some might be directories so we call BackupPC::View::find. |
| 1144 |
# |
| 1145 |
$fio->log("fileListSend: sending file list: " |
| 1146 |
. join(" ", @{$fio->{fileList}})) if ( $fio->{logLevel} >= 4 ); |
| 1147 |
foreach my $name ( @{$fio->{fileList}} ) { |
| 1148 |
$fio->{view}->find($fio->{xfer}{bkupSrcNum}, |
| 1149 |
$fio->{xfer}{bkupSrcShare}, |
| 1150 |
$name, 1, |
| 1151 |
\&fileListEltSend, $fio, $flist, $outputFunc); |
| 1152 |
} |
| 1153 |
} |
| 1154 |
|
| 1155 |
sub finish |
| 1156 |
{ |
| 1157 |
my($fio, $isChild) = @_; |
| 1158 |
|
| 1159 |
# |
| 1160 |
# If we are aborting early, remove the last file since |
| 1161 |
# it was not complete |
| 1162 |
# |
| 1163 |
if ( $isChild && defined($fio->{rxFile}) ) { |
| 1164 |
unlink("$fio->{outDirSh}RStmp") if ( -f "$fio->{outDirSh}RStmp" ); |
| 1165 |
if ( defined($fio->{rxFile}) ) { |
| 1166 |
unlink($fio->{rxOutFile}); |
| 1167 |
$fio->log("finish: removing in-process file $fio->{rxFile}{name}"); |
| 1168 |
} |
| 1169 |
} |
| 1170 |
|
| 1171 |
# |
| 1172 |
# Flush the attributes if this is the child |
| 1173 |
# |
| 1174 |
$fio->attribWrite(undef) if ( $isChild ); |
| 1175 |
} |
| 1176 |
|
| 1177 |
#sub is_tainted |
| 1178 |
#{ |
| 1179 |
# return ! eval { |
| 1180 |
# join('',@_), kill 0; |
| 1181 |
# 1; |
| 1182 |
# }; |
| 1183 |
#} |
| 1184 |
|
| 1185 |
1; |
Parent Directory
| 
Revision Log