/[BackupPC]/trunk/lib/BackupPC/Xfer/Rsync.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /trunk/lib/BackupPC/Xfer/Rsync.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 316 - (hide annotations)
Mon Jan 30 13:37:17 2006 UTC (18 years, 4 months ago) by dpavlin
File size: 14471 byte(s)
 r9152@llin:  dpavlin | 2006-01-30 14:11:45 +0100
 update to upstream 2.1.2

1 dpavlin 1 #============================================================= -*-perl-*-
2     #
3     # BackupPC::Xfer::Rsync package
4     #
5     # DESCRIPTION
6     #
7     # This library defines a BackupPC::Xfer::Rsync class for managing
8     # the rsync-based transport of backup data from the client.
9     #
10     # AUTHOR
11     # Craig Barratt <cbarratt@users.sourceforge.net>
12     #
13     # COPYRIGHT
14     # Copyright (C) 2002-2003 Craig Barratt
15     #
16     # This program is free software; you can redistribute it and/or modify
17     # it under the terms of the GNU General Public License as published by
18     # the Free Software Foundation; either version 2 of the License, or
19     # (at your option) any later version.
20     #
21     # This program is distributed in the hope that it will be useful,
22     # but WITHOUT ANY WARRANTY; without even the implied warranty of
23     # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24     # GNU General Public License for more details.
25     #
26     # You should have received a copy of the GNU General Public License
27     # along with this program; if not, write to the Free Software
28     # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
29     #
30     #========================================================================
31     #
32 dpavlin 316 # Version 2.1.2, released 5 Sep 2005.
33 dpavlin 1 #
34     # See http://backuppc.sourceforge.net.
35     #
36     #========================================================================
37    
38     package BackupPC::Xfer::Rsync;
39    
40     use strict;
41     use BackupPC::View;
42     use BackupPC::Xfer::RsyncFileIO;
43    
44     use vars qw( $RsyncLibOK $RsyncLibErr );
45    
46     BEGIN {
47     eval "use File::RsyncP;";
48     if ( $@ ) {
49     #
50     # Rsync module doesn't exist.
51     #
52     $RsyncLibOK = 0;
53     $RsyncLibErr = "File::RsyncP module doesn't exist";
54     } else {
55     #
56     # Note: also update configure.pl when this version number is changed!
57     #
58 dpavlin 316 if ( $File::RsyncP::VERSION < 0.52 ) {
59 dpavlin 1 $RsyncLibOK = 0;
60 dpavlin 316 $RsyncLibErr = "File::RsyncP module version"
61     . " ($File::RsyncP::VERSION) too old: need 0.52";
62 dpavlin 1 } else {
63     $RsyncLibOK = 1;
64     }
65     }
66     };
67    
68     sub new
69     {
70     my($class, $bpc, $args) = @_;
71    
72     return if ( !$RsyncLibOK );
73     $args ||= {};
74     my $t = bless {
75     bpc => $bpc,
76     conf => { $bpc->Conf },
77     host => "",
78     hostIP => "",
79     shareName => "",
80     badFiles => [],
81    
82     #
83     # Various stats
84     #
85     byteCnt => 0,
86     fileCnt => 0,
87     xferErrCnt => 0,
88     xferBadShareCnt => 0,
89     xferBadFileCnt => 0,
90     xferOK => 0,
91    
92     #
93     # User's args
94     #
95     %$args,
96     }, $class;
97    
98     return $t;
99     }
100    
101     sub args
102     {
103     my($t, $args) = @_;
104    
105     foreach my $arg ( keys(%$args) ) {
106     $t->{$arg} = $args->{$arg};
107     }
108     }
109    
110     sub useTar
111     {
112     return 0;
113     }
114    
115     sub start
116     {
117     my($t) = @_;
118     my $bpc = $t->{bpc};
119     my $conf = $t->{conf};
120     my(@fileList, $rsyncClientCmd, $rsyncArgs, $logMsg,
121     $incrDate, $argList, $fioArgs);
122    
123     #
124     # We add a slash to the share name we pass to rsync
125     #
126     ($t->{shareNameSlash} = "$t->{shareName}/") =~ s{//+$}{/};
127    
128     if ( $t->{type} eq "restore" ) {
129     $rsyncClientCmd = $conf->{RsyncClientRestoreCmd};
130     $rsyncArgs = $conf->{RsyncRestoreArgs};
131     my $remoteDir = "$t->{shareName}/$t->{pathHdrDest}";
132     $remoteDir =~ s{//+}{/}g;
133     $argList = ['--server', @$rsyncArgs, '.', $remoteDir];
134     $fioArgs = {
135     client => $t->{bkupSrcHost},
136     share => $t->{bkupSrcShare},
137     viewNum => $t->{bkupSrcNum},
138     fileList => $t->{fileList},
139     };
140     $logMsg = "restore started below directory $t->{shareName}"
141     . " to host $t->{host}";
142     } else {
143     #
144     # Turn $conf->{BackupFilesOnly} and $conf->{BackupFilesExclude}
145     # into a hash of arrays of files, and $conf->{RsyncShareName}
146     # to an array
147     #
148     $bpc->backupFileConfFix($conf, "RsyncShareName");
149    
150     if ( defined($conf->{BackupFilesOnly}{$t->{shareName}}) ) {
151     my(@inc, @exc, %incDone, %excDone);
152     foreach my $file ( @{$conf->{BackupFilesOnly}{$t->{shareName}}} ) {
153     #
154     # If the user wants to just include /home/craig, then
155     # we need to do create include/exclude pairs at
156     # each level:
157     # --include /home --exclude /*
158     # --include /home/craig --exclude /home/*
159     #
160     # It's more complex if the user wants to include multiple
161     # deep paths. For example, if they want /home/craig and
162     # /var/log, then we need this mouthfull:
163     # --include /home --include /var --exclude /*
164     # --include /home/craig --exclude /home/*
165     # --include /var/log --exclude /var/*
166     #
167     # To make this easier we do all the includes first and all
168     # of the excludes at the end (hopefully they commute).
169     #
170     $file =~ s{/$}{};
171     $file = "/$file";
172     $file =~ s{//+}{/}g;
173     if ( $file eq "/" ) {
174     #
175     # This is a special case: if the user specifies
176     # "/" then just include it and don't exclude "/*".
177     #
178     push(@inc, $file) if ( !$incDone{$file} );
179     next;
180     }
181     my $f = "";
182     while ( $file =~ m{^/([^/]*)(.*)} ) {
183     my $elt = $1;
184     $file = $2;
185     if ( $file eq "/" ) {
186     #
187     # preserve a tailing slash
188     #
189     $file = "";
190     $elt = "$elt/";
191     }
192     push(@exc, "$f/*") if ( !$excDone{"$f/*"} );
193     $excDone{"$f/*"} = 1;
194     $f = "$f/$elt";
195     push(@inc, $f) if ( !$incDone{$f} );
196     $incDone{$f} = 1;
197     }
198     }
199     foreach my $file ( @inc ) {
200     push(@fileList, "--include=$file");
201     }
202     foreach my $file ( @exc ) {
203     push(@fileList, "--exclude=$file");
204     }
205     }
206     if ( defined($conf->{BackupFilesExclude}{$t->{shareName}}) ) {
207     foreach my $file ( @{$conf->{BackupFilesExclude}{$t->{shareName}}} )
208     {
209     #
210     # just append additional exclude lists onto the end
211     #
212     push(@fileList, "--exclude=$file");
213     }
214     }
215     if ( $t->{type} eq "full" ) {
216     if ( $t->{partialNum} ) {
217     $logMsg = "full backup started for directory $t->{shareName};"
218     . " updating partial $t->{partialNum}";
219     } else {
220     $logMsg = "full backup started for directory $t->{shareName}";
221     }
222     } else {
223     $incrDate = $bpc->timeStamp($t->{lastFull} - 3600, 1);
224     $logMsg = "incr backup started back to $incrDate for directory"
225     . " $t->{shareName}";
226     }
227    
228     #
229     # A full dump is implemented with --ignore-times: this causes all
230     # files to be checksummed, even if the attributes are the same.
231     # That way all the file contents are checked, but you get all
232     # the efficiencies of rsync: only files deltas need to be
233     # transferred, even though it is a full dump.
234     #
235     $rsyncArgs = $conf->{RsyncArgs};
236     $rsyncArgs = [@$rsyncArgs, @fileList] if ( @fileList );
237     $rsyncArgs = [@$rsyncArgs, "--ignore-times"]
238     if ( $t->{type} eq "full" );
239     $rsyncClientCmd = $conf->{RsyncClientCmd};
240     $argList = ['--server', '--sender', @$rsyncArgs,
241     '.', $t->{shareNameSlash}];
242     $fioArgs = {
243     client => $t->{client},
244     share => $t->{shareName},
245     viewNum => $t->{lastFullBkupNum},
246     partialNum => $t->{partialNum},
247     };
248     }
249    
250     #
251     # Merge variables into $rsyncClientCmd
252     #
253     my $args = {
254     host => $t->{host},
255     hostIP => $t->{hostIP},
256     client => $t->{client},
257     shareName => $t->{shareName},
258     shareNameSlash => $t->{shareNameSlash},
259     rsyncPath => $conf->{RsyncClientPath},
260     sshPath => $conf->{SshPath},
261     argList => $argList,
262     };
263     $rsyncClientCmd = $bpc->cmdVarSubstitute($rsyncClientCmd, $args);
264    
265     #
266     # Create the Rsync object, and tell it to use our own File::RsyncP::FileIO
267     # module, which handles all the special BackupPC file storage
268     # (compression, mangling, hardlinks, special files, attributes etc).
269     #
270     $t->{rsyncClientCmd} = $rsyncClientCmd;
271     $t->{rs} = File::RsyncP->new({
272     logLevel => $t->{logLevel} || $conf->{RsyncLogLevel},
273     rsyncCmd => sub {
274     $bpc->verbose(0);
275     $bpc->cmdExecOrEval($rsyncClientCmd, $args);
276     },
277     rsyncCmdType => "full",
278     rsyncArgs => $rsyncArgs,
279     timeout => $conf->{ClientTimeout},
280     doPartial => defined($t->{partialNum}) ? 1 : undef,
281     logHandler =>
282     sub {
283     my($str) = @_;
284     $str .= "\n";
285     $t->{XferLOG}->write(\$str);
286     if ( $str =~ /^Remote\[1\]: read errors mapping "(.*)"/ ) {
287     #
288     # Files with read errors (eg: region locked files
289     # on WinXX) are filled with 0 by rsync. Remember
290     # them and delete them later.
291     #
292     my $badFile = $1;
293     $badFile =~ s/^\/+//;
294     push(@{$t->{badFiles}}, {
295     share => $t->{shareName},
296     file => $badFile
297     });
298     }
299     },
300     pidHandler => sub {
301     $t->{pidHandler}(@_);
302     },
303     fio => BackupPC::Xfer::RsyncFileIO->new({
304     xfer => $t,
305     bpc => $t->{bpc},
306     conf => $t->{conf},
307     backups => $t->{backups},
308     logLevel => $t->{logLevel}
309     || $conf->{RsyncLogLevel},
310     logHandler => sub {
311     my($str) = @_;
312     $str .= "\n";
313     $t->{XferLOG}->write(\$str);
314     },
315     cacheCheckProb => $conf->{RsyncCsumCacheVerifyProb},
316     %$fioArgs,
317     }),
318     });
319    
320     delete($t->{_errStr});
321    
322     return $logMsg;
323     }
324    
325     sub run
326     {
327     my($t) = @_;
328     my $rs = $t->{rs};
329     my $conf = $t->{conf};
330     my($remoteSend, $remoteDir, $remoteDirDaemon);
331    
332     alarm($conf->{ClientTimeout});
333     if ( $t->{type} eq "restore" ) {
334     $remoteSend = 0;
335     ($remoteDir = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
336     ($remoteDirDaemon = "$t->{shareName}/$t->{pathHdrDest}") =~ s{//+}{/}g;
337     $remoteDirDaemon = $t->{shareNameSlash}
338     if ( $t->{pathHdrDest} eq ""
339     || $t->{pathHdrDest} eq "/" );
340     } else {
341     $remoteSend = 1;
342     $remoteDir = $t->{shareNameSlash};
343     $remoteDirDaemon = ".";
344     }
345     if ( $t->{XferMethod} eq "rsync" ) {
346     #
347     # Run rsync command
348     #
349     my $str = "Running: "
350     . $t->{bpc}->execCmd2ShellCmd(@{$t->{rsyncClientCmd}})
351     . "\n";
352     $t->{XferLOG}->write(\$str);
353     $rs->remoteStart($remoteSend, $remoteDir);
354     } else {
355     #
356     # Connect to the rsync server
357     #
358     if ( defined(my $err = $rs->serverConnect($t->{hostIP},
359     $conf->{RsyncdClientPort})) ) {
360     $t->{hostError} = $err;
361     my $str = "Error connecting to rsync daemon at $t->{hostIP}"
362     . ":$conf->{RsyncdClientPort}: $err\n";
363     $t->{XferLOG}->write(\$str);
364     return;
365     }
366     #
367     # Pass module name, and follow it with a slash if it already
368     # contains a slash; otherwise just keep the plain module name.
369     #
370     my $module = $t->{shareName};
371     $module = $t->{shareNameSlash} if ( $module =~ /\// );
372     if ( defined(my $err = $rs->serverService($module,
373     $conf->{RsyncdUserName},
374     $conf->{RsyncdPasswd},
375     $conf->{RsyncdAuthRequired})) ) {
376     my $str = "Error connecting to module $module at $t->{hostIP}"
377     . ":$conf->{RsyncdClientPort}: $err\n";
378     $t->{XferLOG}->write(\$str);
379     $t->{hostError} = $err;
380     return;
381     }
382     $rs->serverStart($remoteSend, $remoteDirDaemon);
383     }
384     my $error = $rs->go($t->{shareNameSlash});
385     $rs->serverClose();
386    
387     #
388     # TODO: generate sensible stats
389     #
390     # $rs->{stats}{totalWritten}
391     # $rs->{stats}{totalSize}
392     #
393     my $stats = $rs->statsFinal;
394     if ( !defined($error) && defined($stats) ) {
395     $t->{xferOK} = 1;
396     } else {
397     $t->{xferOK} = 0;
398     }
399     $t->{xferErrCnt} = $stats->{remoteErrCnt}
400     + $stats->{childStats}{errorCnt}
401     + $stats->{parentStats}{errorCnt};
402     $t->{byteCnt} = $stats->{childStats}{TotalFileSize}
403     + $stats->{parentStats}{TotalFileSize};
404     $t->{fileCnt} = $stats->{childStats}{TotalFileCnt}
405     + $stats->{parentStats}{TotalFileCnt};
406     my $str = "Done: $t->{fileCnt} files, $t->{byteCnt} bytes\n";
407     $t->{XferLOG}->write(\$str);
408     #
409     # TODO: get error count, and call fio to get stats...
410     #
411     $t->{hostError} = $error if ( defined($error) );
412    
413     if ( $t->{type} eq "restore" ) {
414     return (
415     $t->{fileCnt},
416     $t->{byteCnt},
417     0,
418     0
419     );
420     } else {
421     return (
422     0,
423     $stats->{childStats}{ExistFileCnt}
424     + $stats->{parentStats}{ExistFileCnt},
425     $stats->{childStats}{ExistFileSize}
426     + $stats->{parentStats}{ExistFileSize},
427     $stats->{childStats}{ExistFileCompSize}
428     + $stats->{parentStats}{ExistFileCompSize},
429     $stats->{childStats}{TotalFileCnt}
430     + $stats->{parentStats}{TotalFileCnt},
431     $stats->{childStats}{TotalFileSize}
432     + $stats->{parentStats}{TotalFileSize},
433     );
434     }
435     }
436    
437     sub abort
438     {
439     my($t, $reason) = @_;
440     my $rs = $t->{rs};
441    
442     $rs->abort($reason);
443     return 1;
444     }
445    
446     sub setSelectMask
447     {
448     my($t, $FDreadRef) = @_;
449     }
450    
451     sub errStr
452     {
453     my($t) = @_;
454    
455     return $RsyncLibErr if ( !defined($t) || ref($t) ne "HASH" );
456     return $t->{_errStr};
457     }
458    
459     sub xferPid
460     {
461     my($t) = @_;
462    
463     return ();
464     }
465    
466     sub logMsg
467     {
468     my($t, $msg) = @_;
469    
470     push(@{$t->{_logMsg}}, $msg);
471     }
472    
473     sub logMsgGet
474     {
475     my($t) = @_;
476    
477     return shift(@{$t->{_logMsg}});
478     }
479    
480     #
481     # Returns a hash ref giving various status information about
482     # the transfer.
483     #
484     sub getStats
485     {
486     my($t) = @_;
487    
488     return { map { $_ => $t->{$_} }
489     qw(byteCnt fileCnt xferErrCnt xferBadShareCnt xferBadFileCnt
490     xferOK hostAbort hostError lastOutputLine)
491     };
492     }
493    
494     sub getBadFiles
495     {
496     my($t) = @_;
497    
498     return @{$t->{badFiles}};
499     }
500    
501     1;

  ViewVC Help
Powered by ViewVC 1.1.26