/[BackupPC]/trunk/bin/BackupPC_tarIncCreate
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /trunk/bin/BackupPC_tarIncCreate

Parent Directory Parent Directory | Revision Log Revision Log


Revision 112 - (show annotations)
Thu Sep 1 20:24:13 2005 UTC (18 years, 8 months ago) by dpavlin
File size: 16696 byte(s)
simplify (not to say refactor ;-) code. Also, make it actually work.
Now it doesn't require files or dirs as last parametar (because we know
that we want all files in that backup).
So, run this script, pipe it to gzip (or bzip2) and you have compressed
archive of one increment.
1 #!/usr/bin/perl
2 #============================================================= -*-perl-*-
3 #
4 # BackupPC_tarIncCreate: create a tar archive of an existing incremental dump
5 #
6 #
7 # DESCRIPTION
8 #
9 # Usage: BackupPC_tarIncCreate [options]
10 #
11 # Flags:
12 # Required options:
13 #
14 # -h host Host from which the tar archive is created.
15 # -n dumpNum Dump number from which the tar archive is created.
16 # A negative number means relative to the end (eg -1
17 # means the most recent dump, -2 2nd most recent etc).
18 # -s shareName Share name from which the tar archive is created.
19 #
20 # Other options:
21 # -t print summary totals
22 # -r pathRemove path prefix that will be replaced with pathAdd
23 # -p pathAdd new path prefix
24 # -b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar)
25 # -w writeBufSz write buffer size (default 1MB)
26 #
27 # The -h, -n and -s options specify which dump is used to generate
28 # the tar archive. The -r and -p options can be used to relocate
29 # the paths in the tar archive so extracted files can be placed
30 # in a location different from their original location.
31 #
32 # AUTHOR
33 # Craig Barratt <cbarratt@users.sourceforge.net>
34 # Ivan Klaric <iklaric@gmail.com>
35 # Dobrica Pavlinusic <dpavlin@rot13.org>
36 #
37 # COPYRIGHT
38 # Copyright (C) 2001-2003 Craig Barratt
39 #
40 # This program is free software; you can redistribute it and/or modify
41 # it under the terms of the GNU General Public License as published by
42 # the Free Software Foundation; either version 2 of the License, or
43 # (at your option) any later version.
44 #
45 # This program is distributed in the hope that it will be useful,
46 # but WITHOUT ANY WARRANTY; without even the implied warranty of
47 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
48 # GNU General Public License for more details.
49 #
50 # You should have received a copy of the GNU General Public License
51 # along with this program; if not, write to the Free Software
52 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
53 #
54 #========================================================================
55 #
56 # Version 2.1.0, released 20 Jun 2004.
57 #
58 # See http://backuppc.sourceforge.net.
59 #
60 #========================================================================
61
62 use strict;
63 no utf8;
64 use lib "__INSTALLDIR__/lib";
65 use File::Path;
66 use Getopt::Std;
67 use DBI;
68 use BackupPC::Lib;
69 use BackupPC::Attrib qw(:all);
70 use BackupPC::FileZIO;
71 use BackupPC::View;
72 use BackupPC::SearchLib;
73 use Time::HiRes qw/time/;
74 use POSIX qw/strftime/;
75 use Data::Dumper; ### FIXME
76
77 die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) );
78 my $TopDir = $bpc->TopDir();
79 my $BinDir = $bpc->BinDir();
80 my %Conf = $bpc->Conf();
81 my %opts;
82 my $in_backup_increment;
83
84
85 if ( !getopts("th:n:p:r:s:b:w:", \%opts) ) {
86 print STDERR <<EOF;
87 usage: $0 [options]
88 Required options:
89 -h host host from which the tar archive is created
90 -n dumpNum dump number from which the tar archive is created
91 A negative number means relative to the end (eg -1
92 means the most recent dump, -2 2nd most recent etc).
93 -s shareName share name from which the tar archive is created
94
95 Other options:
96 -t print summary totals
97 -r pathRemove path prefix that will be replaced with pathAdd
98 -p pathAdd new path prefix
99 -b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar)
100 -w writeBufSz write buffer size (default 1048576 = 1MB)
101 EOF
102 exit(1);
103 }
104
105 if ( $opts{h} !~ /^([\w\.\s-]+)$/ ) {
106 print(STDERR "$0: bad host name '$opts{h}'\n");
107 exit(1);
108 }
109 my $Host = $opts{h};
110
111 if ( $opts{n} !~ /^(-?\d+)$/ ) {
112 print(STDERR "$0: bad dump number '$opts{n}'\n");
113 exit(1);
114 }
115 my $Num = $opts{n};
116
117 my @Backups = $bpc->BackupInfoRead($Host);
118 my $FileCnt = 0;
119 my $ByteCnt = 0;
120 my $DirCnt = 0;
121 my $SpecialCnt = 0;
122 my $ErrorCnt = 0;
123
124 my $i;
125 $Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 );
126 for ( $i = 0 ; $i < @Backups ; $i++ ) {
127 last if ( $Backups[$i]{num} == $Num );
128 }
129 if ( $i >= @Backups ) {
130 print(STDERR "$0: bad backup number $Num for host $Host\n");
131 exit(1);
132 }
133
134 my $PathRemove = $1 if ( $opts{r} =~ /(.+)/ );
135 my $PathAdd = $1 if ( $opts{p} =~ /(.+)/ );
136 if ( $opts{s} !~ /^([\w\s\.\/\$-]+)$/ && $opts{s} ne "*" ) {
137 print(STDERR "$0: bad share name '$opts{s}'\n");
138 exit(1);
139 }
140 our $ShareName = $opts{s};
141 our $view = BackupPC::View->new($bpc, $Host, \@Backups);
142
143 #
144 # This constant and the line of code below that uses it are borrowed
145 # from Archive::Tar. Thanks to Calle Dybedahl and Stephen Zander.
146 # See www.cpan.org.
147 #
148 # Archive::Tar is Copyright 1997 Calle Dybedahl. All rights reserved.
149 # Copyright 1998 Stephen Zander. All rights reserved.
150 #
151 my $tar_pack_header
152 = 'a100 a8 a8 a8 a12 a12 A8 a1 a100 a6 a2 a32 a32 a8 a8 a155 x12';
153 my $tar_header_length = 512;
154
155 my $BufSize = $opts{w} || 1048576; # 1MB or 2^20
156 my $WriteBuf = "";
157 my $WriteBufSz = ($opts{b} || 20) * $tar_header_length;
158
159 my(%UidCache, %GidCache);
160 my(%HardLinkExtraFiles, @HardLinks);
161
162 #
163 # Write out all the requested files/directories
164 #
165 binmode(STDOUT);
166 my $fh = *STDOUT;
167
168 if (seedCache($Host, $ShareName, $Num)) {
169 archiveWrite($fh, '/');
170 archiveWriteHardLinks($fh);
171 } else {
172 print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n";
173 }
174
175 #
176 # Finish with two null 512 byte headers, and then round out a full
177 # block.
178 #
179 my $data = "\0" x ($tar_header_length * 2);
180 TarWrite($fh, \$data);
181 TarWrite($fh, undef);
182
183 #
184 # print out totals if requested
185 #
186 if ( $opts{t} ) {
187 print STDERR "Done: $FileCnt files, $ByteCnt bytes, $DirCnt dirs,",
188 " $SpecialCnt specials, $ErrorCnt errors\n";
189 }
190 if ( $ErrorCnt && !$FileCnt && !$DirCnt ) {
191 #
192 # Got errors, with no files or directories; exit with non-zero
193 # status
194 #
195 exit(1);
196 }
197 exit(0);
198
199 ###########################################################################
200 # Subroutines
201 ###########################################################################
202
203 sub archiveWrite
204 {
205 my($fh, $dir, $tarPathOverride) = @_;
206
207 if ( $dir =~ m{(^|/)\.\.(/|$)} ) {
208 print(STDERR "$0: bad directory '$dir'\n");
209 $ErrorCnt++;
210 return;
211 }
212 $dir = "/" if ( $dir eq "." );
213 #print(STDERR "calling find with $Num, $ShareName, $dir\n");
214
215 if ( $view->find($Num, $ShareName, $dir, 0, \&TarWriteFile,
216 $fh, $tarPathOverride) < 0 ) {
217 print(STDERR "$0: bad share or directory '$ShareName/$dir'\n");
218 $ErrorCnt++;
219 return;
220 }
221 }
222
223 #
224 # Write out any hardlinks (if any)
225 #
226 sub archiveWriteHardLinks
227 {
228 my $fh = @_;
229 foreach my $hdr ( @HardLinks ) {
230 $hdr->{size} = 0;
231 if ( defined($PathRemove)
232 && substr($hdr->{linkname}, 0, length($PathRemove)+1)
233 eq ".$PathRemove" ) {
234 substr($hdr->{linkname}, 0, length($PathRemove)+1) = ".$PathAdd";
235 }
236 TarWriteFileInfo($fh, $hdr);
237 }
238 @HardLinks = ();
239 %HardLinkExtraFiles = ();
240 }
241
242 sub UidLookup
243 {
244 my($uid) = @_;
245
246 $UidCache{$uid} = (getpwuid($uid))[0] if ( !exists($UidCache{$uid}) );
247 return $UidCache{$uid};
248 }
249
250 sub GidLookup
251 {
252 my($gid) = @_;
253
254 $GidCache{$gid} = (getgrgid($gid))[0] if ( !exists($GidCache{$gid}) );
255 return $GidCache{$gid};
256 }
257
258 sub TarWrite
259 {
260 my($fh, $dataRef) = @_;
261
262 if ( !defined($dataRef) ) {
263 #
264 # do flush by padding to a full $WriteBufSz
265 #
266 my $data = "\0" x ($WriteBufSz - length($WriteBuf));
267 $dataRef = \$data;
268 }
269 if ( length($WriteBuf) + length($$dataRef) < $WriteBufSz ) {
270 #
271 # just buffer and return
272 #
273 $WriteBuf .= $$dataRef;
274 return;
275 }
276 my $done = $WriteBufSz - length($WriteBuf);
277 if ( syswrite($fh, $WriteBuf . substr($$dataRef, 0, $done))
278 != $WriteBufSz ) {
279 print(STDERR "Unable to write to output file ($!)\n");
280 exit(1);
281 }
282 while ( $done + $WriteBufSz <= length($$dataRef) ) {
283 if ( syswrite($fh, substr($$dataRef, $done, $WriteBufSz))
284 != $WriteBufSz ) {
285 print(STDERR "Unable to write to output file ($!)\n");
286 exit(1);
287 }
288 $done += $WriteBufSz;
289 }
290 $WriteBuf = substr($$dataRef, $done);
291 }
292
293 sub TarWritePad
294 {
295 my($fh, $size) = @_;
296
297 if ( $size % $tar_header_length ) {
298 my $data = "\0" x ($tar_header_length - ($size % $tar_header_length));
299 TarWrite($fh, \$data);
300 }
301 }
302
303 sub TarWriteHeader
304 {
305 my($fh, $hdr) = @_;
306
307 $hdr->{uname} = UidLookup($hdr->{uid}) if ( !defined($hdr->{uname}) );
308 $hdr->{gname} = GidLookup($hdr->{gid}) if ( !defined($hdr->{gname}) );
309 my $devmajor = defined($hdr->{devmajor}) ? sprintf("%07o", $hdr->{devmajor})
310 : "";
311 my $devminor = defined($hdr->{devminor}) ? sprintf("%07o", $hdr->{devminor})
312 : "";
313 my $sizeStr;
314 if ( $hdr->{size} >= 2 * 65536 * 65536 ) {
315 #
316 # GNU extension for files >= 8GB: send size in big-endian binary
317 #
318 $sizeStr = pack("c4 N N", 0x80, 0, 0, 0,
319 $hdr->{size} / (65536 * 65536),
320 $hdr->{size} % (65536 * 65536));
321 } elsif ( $hdr->{size} >= 1 * 65536 * 65536 ) {
322 #
323 # sprintf octal only handles up to 2^32 - 1
324 #
325 $sizeStr = sprintf("%03o", $hdr->{size} / (1 << 24))
326 . sprintf("%08o", $hdr->{size} % (1 << 24));
327 } else {
328 $sizeStr = sprintf("%011o", $hdr->{size});
329 }
330 my $data = pack($tar_pack_header,
331 substr($hdr->{name}, 0, 99),
332 sprintf("%07o", $hdr->{mode}),
333 sprintf("%07o", $hdr->{uid}),
334 sprintf("%07o", $hdr->{gid}),
335 $sizeStr,
336 sprintf("%011o", $hdr->{mtime}),
337 "", #checksum field - space padded by pack("A8")
338 $hdr->{type},
339 substr($hdr->{linkname}, 0, 99),
340 $hdr->{magic} || 'ustar ',
341 $hdr->{version} || ' ',
342 $hdr->{uname},
343 $hdr->{gname},
344 $devmajor,
345 $devminor,
346 "" # prefix is empty
347 );
348 substr($data, 148, 7) = sprintf("%06o\0", unpack("%16C*",$data));
349 TarWrite($fh, \$data);
350 }
351
352 sub TarWriteFileInfo
353 {
354 my($fh, $hdr) = @_;
355
356 #
357 # Handle long link names (symbolic links)
358 #
359 if ( length($hdr->{linkname}) > 99 ) {
360 my %h;
361 my $data = $hdr->{linkname} . "\0";
362 $h{name} = "././\@LongLink";
363 $h{type} = "K";
364 $h{size} = length($data);
365 TarWriteHeader($fh, \%h);
366 TarWrite($fh, \$data);
367 TarWritePad($fh, length($data));
368 }
369 #
370 # Handle long file names
371 #
372 if ( length($hdr->{name}) > 99 ) {
373 my %h;
374 my $data = $hdr->{name} . "\0";
375 $h{name} = "././\@LongLink";
376 $h{type} = "L";
377 $h{size} = length($data);
378 TarWriteHeader($fh, \%h);
379 TarWrite($fh, \$data);
380 TarWritePad($fh, length($data));
381 }
382 TarWriteHeader($fh, $hdr);
383 }
384
385 #
386 # seed cache of files in this increment
387 #
388 sub seedCache($$$) {
389 my ($host, $share, $dumpNo) = @_;
390
391 my $dsn = $Conf{SearchDSN};
392 my $db_user = $Conf{SearchUser} || '';
393
394 print STDERR curr_time(), "getting files for $host:$share increment $dumpNo...";
395 my $sql = q{
396 SELECT path
397 FROM files
398 JOIN shares on shares.id = shareid
399 JOIN hosts on hosts.id = shares.hostid
400 WHERE hosts.name = ? and shares.name = ? and backupnum = ?
401 };
402
403 my $dbh = DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1} );
404 my $sth = $dbh->prepare($sql);
405 $sth->execute($host, $share, $dumpNo);
406 my $count = $sth->rows;
407 print STDERR " found $count items\n";
408 while (my $row = $sth->fetchrow_arrayref) {
409 print STDERR "+ ", $row->[0],"\n";
410 $in_backup_increment->{ $row->[0] }++;
411 }
412
413 $sth->finish();
414 $dbh->disconnect();
415
416 return $count;
417 }
418
419 my $Attr;
420 my $AttrDir;
421
422 sub TarWriteFile
423 {
424 my($hdr, $fh, $tarPathOverride) = @_;
425
426 my $tarPath = $hdr->{relPath};
427 $tarPath = $tarPathOverride if ( defined($tarPathOverride) );
428
429 $tarPath =~ s{//+}{/}g;
430
431 #print STDERR "? $tarPath\n";
432 return unless ($in_backup_increment->{$tarPath});
433 print STDERR "A $tarPath\n";
434
435 if ( defined($PathRemove)
436 && substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) {
437 substr($tarPath, 0, length($PathRemove)) = $PathAdd;
438 }
439 $tarPath = "./" . $tarPath if ( $tarPath !~ /^\.\// );
440 $tarPath =~ s{//+}{/}g;
441 $hdr->{name} = $tarPath;
442
443 if ( $hdr->{type} == BPC_FTYPE_DIR ) {
444 #
445 # Directory: just write the header
446 #
447
448
449 $hdr->{name} .= "/" if ( $hdr->{name} !~ m{/$} );
450 TarWriteFileInfo($fh, $hdr);
451 $DirCnt++;
452 } elsif ( $hdr->{type} == BPC_FTYPE_FILE ) {
453 #
454 # Regular file: write the header and file
455 #
456 my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress});
457 if ( !defined($f) ) {
458 print(STDERR "Unable to open file $hdr->{fullPath}\n");
459 $ErrorCnt++;
460 return;
461 }
462 TarWriteFileInfo($fh, $hdr);
463 my($data, $size);
464 while ( $f->read(\$data, $BufSize) > 0 ) {
465 TarWrite($fh, \$data);
466 $size += length($data);
467 }
468 $f->close;
469 TarWritePad($fh, $size);
470 $FileCnt++;
471 $ByteCnt += $size;
472 } elsif ( $hdr->{type} == BPC_FTYPE_HARDLINK ) {
473 #
474 # Hardlink file: either write a hardlink or the complete file
475 # depending upon whether the linked-to file will be written
476 # to the archive.
477 #
478 # Start by reading the contents of the link.
479 #
480 my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress});
481 if ( !defined($f) ) {
482 print(STDERR "Unable to open file $hdr->{fullPath}\n");
483 $ErrorCnt++;
484 return;
485 }
486 my $data;
487 while ( $f->read(\$data, $BufSize) > 0 ) {
488 $hdr->{linkname} .= $data;
489 }
490 $f->close;
491 my $done = 0;
492 my $name = $hdr->{linkname};
493 $name =~ s{^\./}{/};
494 if ( $HardLinkExtraFiles{$name} ) {
495 #
496 # Target file will be or was written, so just remember
497 # the hardlink so we can dump it later.
498 #
499 push(@HardLinks, $hdr);
500 $SpecialCnt++;
501 } else {
502 #
503 # Have to dump the original file. Just call the top-level
504 # routine, so that we save the hassle of dealing with
505 # mangling, merging and attributes.
506 #
507 $HardLinkExtraFiles{$hdr->{linkname}} = 1;
508 archiveWrite($fh, $hdr->{linkname}, $hdr->{name});
509 }
510 } elsif ( $hdr->{type} == BPC_FTYPE_SYMLINK ) {
511 #
512 # Symbolic link: read the symbolic link contents into the header
513 # and write the header.
514 #
515 my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress});
516 if ( !defined($f) ) {
517 print(STDERR "Unable to open symlink file $hdr->{fullPath}\n");
518 $ErrorCnt++;
519 return;
520 }
521 my $data;
522 while ( $f->read(\$data, $BufSize) > 0 ) {
523 $hdr->{linkname} .= $data;
524 }
525 $f->close;
526 $hdr->{size} = 0;
527 TarWriteFileInfo($fh, $hdr);
528 $SpecialCnt++;
529 } elsif ( $hdr->{type} == BPC_FTYPE_CHARDEV
530 || $hdr->{type} == BPC_FTYPE_BLOCKDEV
531 || $hdr->{type} == BPC_FTYPE_FIFO ) {
532 #
533 # Special files: for char and block special we read the
534 # major and minor numbers from a plain file.
535 #
536 if ( $hdr->{type} != BPC_FTYPE_FIFO ) {
537 my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0,
538 $hdr->{compress});
539 my $data;
540 if ( !defined($f) || $f->read(\$data, $BufSize) < 0 ) {
541 print(STDERR "Unable to open/read char/block special file"
542 . " $hdr->{fullPath}\n");
543 $f->close if ( defined($f) );
544 $ErrorCnt++;
545 return;
546 }
547 $f->close;
548 if ( $data =~ /(\d+),(\d+)/ ) {
549 $hdr->{devmajor} = $1;
550 $hdr->{devminor} = $2;
551 }
552 }
553 $hdr->{size} = 0;
554 TarWriteFileInfo($fh, $hdr);
555 $SpecialCnt++;
556 } else {
557 print(STDERR "Got unknown type $hdr->{type} for $hdr->{name}\n");
558 $ErrorCnt++;
559 }
560 }
561
562 my $t_fmt = '%Y-%m-%d %H:%M:%S';
563 sub curr_time {
564 return strftime($t_fmt,localtime());
565 }

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26