/[BackupPC]/trunk/bin/BackupPC_tarIncCreate
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /trunk/bin/BackupPC_tarIncCreate

Parent Directory Parent Directory | Revision Log Revision Log


Revision 112 - (hide annotations)
Thu Sep 1 20:24:13 2005 UTC (18 years, 8 months ago) by dpavlin
File size: 16696 byte(s)
simplify (not to say refactor ;-) code. Also, make it actually work.
Now it doesn't require files or dirs as last parametar (because we know
that we want all files in that backup).
So, run this script, pipe it to gzip (or bzip2) and you have compressed
archive of one increment.
1 dpavlin 100 #!/usr/bin/perl
2     #============================================================= -*-perl-*-
3     #
4 dpavlin 109 # BackupPC_tarIncCreate: create a tar archive of an existing incremental dump
5     #
6 dpavlin 100 #
7     # DESCRIPTION
8     #
9 dpavlin 112 # Usage: BackupPC_tarIncCreate [options]
10 dpavlin 100 #
11     # Flags:
12     # Required options:
13     #
14     # -h host Host from which the tar archive is created.
15     # -n dumpNum Dump number from which the tar archive is created.
16     # A negative number means relative to the end (eg -1
17     # means the most recent dump, -2 2nd most recent etc).
18     # -s shareName Share name from which the tar archive is created.
19     #
20     # Other options:
21     # -t print summary totals
22     # -r pathRemove path prefix that will be replaced with pathAdd
23     # -p pathAdd new path prefix
24     # -b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar)
25     # -w writeBufSz write buffer size (default 1MB)
26     #
27     # The -h, -n and -s options specify which dump is used to generate
28     # the tar archive. The -r and -p options can be used to relocate
29     # the paths in the tar archive so extracted files can be placed
30     # in a location different from their original location.
31     #
32     # AUTHOR
33     # Craig Barratt <cbarratt@users.sourceforge.net>
34 dpavlin 112 # Ivan Klaric <iklaric@gmail.com>
35     # Dobrica Pavlinusic <dpavlin@rot13.org>
36 dpavlin 100 #
37     # COPYRIGHT
38     # Copyright (C) 2001-2003 Craig Barratt
39     #
40     # This program is free software; you can redistribute it and/or modify
41     # it under the terms of the GNU General Public License as published by
42     # the Free Software Foundation; either version 2 of the License, or
43     # (at your option) any later version.
44     #
45     # This program is distributed in the hope that it will be useful,
46     # but WITHOUT ANY WARRANTY; without even the implied warranty of
47     # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
48     # GNU General Public License for more details.
49     #
50     # You should have received a copy of the GNU General Public License
51     # along with this program; if not, write to the Free Software
52     # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
53     #
54     #========================================================================
55     #
56     # Version 2.1.0, released 20 Jun 2004.
57     #
58     # See http://backuppc.sourceforge.net.
59     #
60     #========================================================================
61    
62     use strict;
63     no utf8;
64     use lib "__INSTALLDIR__/lib";
65     use File::Path;
66     use Getopt::Std;
67     use DBI;
68     use BackupPC::Lib;
69     use BackupPC::Attrib qw(:all);
70     use BackupPC::FileZIO;
71     use BackupPC::View;
72     use BackupPC::SearchLib;
73 dpavlin 112 use Time::HiRes qw/time/;
74     use POSIX qw/strftime/;
75     use Data::Dumper; ### FIXME
76 dpavlin 100
77     die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) );
78     my $TopDir = $bpc->TopDir();
79     my $BinDir = $bpc->BinDir();
80     my %Conf = $bpc->Conf();
81     my %opts;
82 dpavlin 112 my $in_backup_increment;
83 dpavlin 100
84 dpavlin 112
85     if ( !getopts("th:n:p:r:s:b:w:", \%opts) ) {
86 dpavlin 100 print STDERR <<EOF;
87 dpavlin 112 usage: $0 [options]
88 dpavlin 100 Required options:
89     -h host host from which the tar archive is created
90     -n dumpNum dump number from which the tar archive is created
91     A negative number means relative to the end (eg -1
92     means the most recent dump, -2 2nd most recent etc).
93     -s shareName share name from which the tar archive is created
94    
95     Other options:
96     -t print summary totals
97     -r pathRemove path prefix that will be replaced with pathAdd
98     -p pathAdd new path prefix
99     -b BLOCKS BLOCKS x 512 bytes per record (default 20; same as tar)
100     -w writeBufSz write buffer size (default 1048576 = 1MB)
101     EOF
102     exit(1);
103     }
104    
105     if ( $opts{h} !~ /^([\w\.\s-]+)$/ ) {
106     print(STDERR "$0: bad host name '$opts{h}'\n");
107     exit(1);
108     }
109     my $Host = $opts{h};
110    
111     if ( $opts{n} !~ /^(-?\d+)$/ ) {
112     print(STDERR "$0: bad dump number '$opts{n}'\n");
113     exit(1);
114     }
115     my $Num = $opts{n};
116    
117     my @Backups = $bpc->BackupInfoRead($Host);
118     my $FileCnt = 0;
119     my $ByteCnt = 0;
120     my $DirCnt = 0;
121     my $SpecialCnt = 0;
122     my $ErrorCnt = 0;
123    
124     my $i;
125     $Num = $Backups[@Backups + $Num]{num} if ( -@Backups <= $Num && $Num < 0 );
126     for ( $i = 0 ; $i < @Backups ; $i++ ) {
127     last if ( $Backups[$i]{num} == $Num );
128     }
129     if ( $i >= @Backups ) {
130     print(STDERR "$0: bad backup number $Num for host $Host\n");
131     exit(1);
132     }
133    
134     my $PathRemove = $1 if ( $opts{r} =~ /(.+)/ );
135     my $PathAdd = $1 if ( $opts{p} =~ /(.+)/ );
136     if ( $opts{s} !~ /^([\w\s\.\/\$-]+)$/ && $opts{s} ne "*" ) {
137     print(STDERR "$0: bad share name '$opts{s}'\n");
138     exit(1);
139     }
140     our $ShareName = $opts{s};
141     our $view = BackupPC::View->new($bpc, $Host, \@Backups);
142    
143     #
144     # This constant and the line of code below that uses it are borrowed
145     # from Archive::Tar. Thanks to Calle Dybedahl and Stephen Zander.
146     # See www.cpan.org.
147     #
148     # Archive::Tar is Copyright 1997 Calle Dybedahl. All rights reserved.
149     # Copyright 1998 Stephen Zander. All rights reserved.
150     #
151     my $tar_pack_header
152     = 'a100 a8 a8 a8 a12 a12 A8 a1 a100 a6 a2 a32 a32 a8 a8 a155 x12';
153     my $tar_header_length = 512;
154    
155     my $BufSize = $opts{w} || 1048576; # 1MB or 2^20
156     my $WriteBuf = "";
157     my $WriteBufSz = ($opts{b} || 20) * $tar_header_length;
158    
159     my(%UidCache, %GidCache);
160     my(%HardLinkExtraFiles, @HardLinks);
161    
162     #
163     # Write out all the requested files/directories
164     #
165     binmode(STDOUT);
166     my $fh = *STDOUT;
167 dpavlin 112
168     if (seedCache($Host, $ShareName, $Num)) {
169     archiveWrite($fh, '/');
170     archiveWriteHardLinks($fh);
171 dpavlin 100 } else {
172 dpavlin 112 print STDERR "NOTE: no files found for $Host:$ShareName, increment $Num\n";
173 dpavlin 100 }
174    
175     #
176     # Finish with two null 512 byte headers, and then round out a full
177     # block.
178     #
179     my $data = "\0" x ($tar_header_length * 2);
180     TarWrite($fh, \$data);
181     TarWrite($fh, undef);
182    
183     #
184     # print out totals if requested
185     #
186     if ( $opts{t} ) {
187     print STDERR "Done: $FileCnt files, $ByteCnt bytes, $DirCnt dirs,",
188     " $SpecialCnt specials, $ErrorCnt errors\n";
189     }
190     if ( $ErrorCnt && !$FileCnt && !$DirCnt ) {
191     #
192     # Got errors, with no files or directories; exit with non-zero
193     # status
194     #
195     exit(1);
196     }
197     exit(0);
198    
199     ###########################################################################
200     # Subroutines
201     ###########################################################################
202    
203     sub archiveWrite
204     {
205     my($fh, $dir, $tarPathOverride) = @_;
206    
207     if ( $dir =~ m{(^|/)\.\.(/|$)} ) {
208     print(STDERR "$0: bad directory '$dir'\n");
209     $ErrorCnt++;
210     return;
211     }
212     $dir = "/" if ( $dir eq "." );
213     #print(STDERR "calling find with $Num, $ShareName, $dir\n");
214    
215     if ( $view->find($Num, $ShareName, $dir, 0, \&TarWriteFile,
216     $fh, $tarPathOverride) < 0 ) {
217     print(STDERR "$0: bad share or directory '$ShareName/$dir'\n");
218     $ErrorCnt++;
219     return;
220     }
221     }
222    
223     #
224     # Write out any hardlinks (if any)
225     #
226     sub archiveWriteHardLinks
227     {
228     my $fh = @_;
229     foreach my $hdr ( @HardLinks ) {
230     $hdr->{size} = 0;
231     if ( defined($PathRemove)
232     && substr($hdr->{linkname}, 0, length($PathRemove)+1)
233     eq ".$PathRemove" ) {
234     substr($hdr->{linkname}, 0, length($PathRemove)+1) = ".$PathAdd";
235     }
236     TarWriteFileInfo($fh, $hdr);
237     }
238     @HardLinks = ();
239     %HardLinkExtraFiles = ();
240     }
241    
242     sub UidLookup
243     {
244     my($uid) = @_;
245    
246     $UidCache{$uid} = (getpwuid($uid))[0] if ( !exists($UidCache{$uid}) );
247     return $UidCache{$uid};
248     }
249    
250     sub GidLookup
251     {
252     my($gid) = @_;
253    
254     $GidCache{$gid} = (getgrgid($gid))[0] if ( !exists($GidCache{$gid}) );
255     return $GidCache{$gid};
256     }
257    
258     sub TarWrite
259     {
260     my($fh, $dataRef) = @_;
261    
262     if ( !defined($dataRef) ) {
263     #
264     # do flush by padding to a full $WriteBufSz
265     #
266     my $data = "\0" x ($WriteBufSz - length($WriteBuf));
267     $dataRef = \$data;
268     }
269     if ( length($WriteBuf) + length($$dataRef) < $WriteBufSz ) {
270     #
271     # just buffer and return
272     #
273     $WriteBuf .= $$dataRef;
274     return;
275     }
276     my $done = $WriteBufSz - length($WriteBuf);
277     if ( syswrite($fh, $WriteBuf . substr($$dataRef, 0, $done))
278     != $WriteBufSz ) {
279     print(STDERR "Unable to write to output file ($!)\n");
280     exit(1);
281     }
282     while ( $done + $WriteBufSz <= length($$dataRef) ) {
283     if ( syswrite($fh, substr($$dataRef, $done, $WriteBufSz))
284     != $WriteBufSz ) {
285     print(STDERR "Unable to write to output file ($!)\n");
286     exit(1);
287     }
288     $done += $WriteBufSz;
289     }
290     $WriteBuf = substr($$dataRef, $done);
291     }
292    
293     sub TarWritePad
294     {
295     my($fh, $size) = @_;
296    
297     if ( $size % $tar_header_length ) {
298     my $data = "\0" x ($tar_header_length - ($size % $tar_header_length));
299     TarWrite($fh, \$data);
300     }
301     }
302    
303     sub TarWriteHeader
304     {
305     my($fh, $hdr) = @_;
306    
307     $hdr->{uname} = UidLookup($hdr->{uid}) if ( !defined($hdr->{uname}) );
308     $hdr->{gname} = GidLookup($hdr->{gid}) if ( !defined($hdr->{gname}) );
309     my $devmajor = defined($hdr->{devmajor}) ? sprintf("%07o", $hdr->{devmajor})
310     : "";
311     my $devminor = defined($hdr->{devminor}) ? sprintf("%07o", $hdr->{devminor})
312     : "";
313     my $sizeStr;
314     if ( $hdr->{size} >= 2 * 65536 * 65536 ) {
315     #
316     # GNU extension for files >= 8GB: send size in big-endian binary
317     #
318     $sizeStr = pack("c4 N N", 0x80, 0, 0, 0,
319     $hdr->{size} / (65536 * 65536),
320     $hdr->{size} % (65536 * 65536));
321     } elsif ( $hdr->{size} >= 1 * 65536 * 65536 ) {
322     #
323     # sprintf octal only handles up to 2^32 - 1
324     #
325     $sizeStr = sprintf("%03o", $hdr->{size} / (1 << 24))
326     . sprintf("%08o", $hdr->{size} % (1 << 24));
327     } else {
328     $sizeStr = sprintf("%011o", $hdr->{size});
329     }
330     my $data = pack($tar_pack_header,
331     substr($hdr->{name}, 0, 99),
332     sprintf("%07o", $hdr->{mode}),
333     sprintf("%07o", $hdr->{uid}),
334     sprintf("%07o", $hdr->{gid}),
335     $sizeStr,
336     sprintf("%011o", $hdr->{mtime}),
337     "", #checksum field - space padded by pack("A8")
338     $hdr->{type},
339     substr($hdr->{linkname}, 0, 99),
340     $hdr->{magic} || 'ustar ',
341     $hdr->{version} || ' ',
342     $hdr->{uname},
343     $hdr->{gname},
344     $devmajor,
345     $devminor,
346     "" # prefix is empty
347     );
348     substr($data, 148, 7) = sprintf("%06o\0", unpack("%16C*",$data));
349     TarWrite($fh, \$data);
350     }
351    
352     sub TarWriteFileInfo
353     {
354     my($fh, $hdr) = @_;
355    
356     #
357     # Handle long link names (symbolic links)
358     #
359     if ( length($hdr->{linkname}) > 99 ) {
360     my %h;
361     my $data = $hdr->{linkname} . "\0";
362     $h{name} = "././\@LongLink";
363     $h{type} = "K";
364     $h{size} = length($data);
365     TarWriteHeader($fh, \%h);
366     TarWrite($fh, \$data);
367     TarWritePad($fh, length($data));
368     }
369     #
370     # Handle long file names
371     #
372     if ( length($hdr->{name}) > 99 ) {
373     my %h;
374     my $data = $hdr->{name} . "\0";
375     $h{name} = "././\@LongLink";
376     $h{type} = "L";
377     $h{size} = length($data);
378     TarWriteHeader($fh, \%h);
379     TarWrite($fh, \$data);
380     TarWritePad($fh, length($data));
381     }
382     TarWriteHeader($fh, $hdr);
383     }
384    
385     #
386 dpavlin 112 # seed cache of files in this increment
387 dpavlin 100 #
388 dpavlin 112 sub seedCache($$$) {
389     my ($host, $share, $dumpNo) = @_;
390    
391 dpavlin 100 my $dsn = $Conf{SearchDSN};
392     my $db_user = $Conf{SearchUser} || '';
393    
394 dpavlin 112 print STDERR curr_time(), "getting files for $host:$share increment $dumpNo...";
395     my $sql = q{
396     SELECT path
397     FROM files
398     JOIN shares on shares.id = shareid
399     JOIN hosts on hosts.id = shares.hostid
400     WHERE hosts.name = ? and shares.name = ? and backupnum = ?
401     };
402 dpavlin 100
403 dpavlin 112 my $dbh = DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1} );
404     my $sth = $dbh->prepare($sql);
405     $sth->execute($host, $share, $dumpNo);
406     my $count = $sth->rows;
407     print STDERR " found $count items\n";
408     while (my $row = $sth->fetchrow_arrayref) {
409     print STDERR "+ ", $row->[0],"\n";
410     $in_backup_increment->{ $row->[0] }++;
411 dpavlin 100 }
412 dpavlin 112
413     $sth->finish();
414     $dbh->disconnect();
415 dpavlin 100
416 dpavlin 112 return $count;
417 dpavlin 100 }
418    
419     my $Attr;
420     my $AttrDir;
421    
422     sub TarWriteFile
423     {
424     my($hdr, $fh, $tarPathOverride) = @_;
425    
426     my $tarPath = $hdr->{relPath};
427     $tarPath = $tarPathOverride if ( defined($tarPathOverride) );
428    
429     $tarPath =~ s{//+}{/}g;
430 dpavlin 112
431     #print STDERR "? $tarPath\n";
432     return unless ($in_backup_increment->{$tarPath});
433     print STDERR "A $tarPath\n";
434    
435 dpavlin 100 if ( defined($PathRemove)
436     && substr($tarPath, 0, length($PathRemove)) eq $PathRemove ) {
437     substr($tarPath, 0, length($PathRemove)) = $PathAdd;
438     }
439     $tarPath = "./" . $tarPath if ( $tarPath !~ /^\.\// );
440     $tarPath =~ s{//+}{/}g;
441     $hdr->{name} = $tarPath;
442    
443     if ( $hdr->{type} == BPC_FTYPE_DIR ) {
444     #
445     # Directory: just write the header
446     #
447    
448    
449     $hdr->{name} .= "/" if ( $hdr->{name} !~ m{/$} );
450 dpavlin 112 TarWriteFileInfo($fh, $hdr);
451     $DirCnt++;
452 dpavlin 100 } elsif ( $hdr->{type} == BPC_FTYPE_FILE ) {
453     #
454     # Regular file: write the header and file
455     #
456     my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress});
457     if ( !defined($f) ) {
458     print(STDERR "Unable to open file $hdr->{fullPath}\n");
459     $ErrorCnt++;
460     return;
461     }
462     TarWriteFileInfo($fh, $hdr);
463     my($data, $size);
464     while ( $f->read(\$data, $BufSize) > 0 ) {
465     TarWrite($fh, \$data);
466     $size += length($data);
467     }
468     $f->close;
469     TarWritePad($fh, $size);
470     $FileCnt++;
471     $ByteCnt += $size;
472     } elsif ( $hdr->{type} == BPC_FTYPE_HARDLINK ) {
473     #
474     # Hardlink file: either write a hardlink or the complete file
475     # depending upon whether the linked-to file will be written
476     # to the archive.
477     #
478     # Start by reading the contents of the link.
479     #
480     my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress});
481     if ( !defined($f) ) {
482     print(STDERR "Unable to open file $hdr->{fullPath}\n");
483     $ErrorCnt++;
484     return;
485     }
486     my $data;
487     while ( $f->read(\$data, $BufSize) > 0 ) {
488     $hdr->{linkname} .= $data;
489     }
490     $f->close;
491     my $done = 0;
492     my $name = $hdr->{linkname};
493     $name =~ s{^\./}{/};
494     if ( $HardLinkExtraFiles{$name} ) {
495     #
496     # Target file will be or was written, so just remember
497     # the hardlink so we can dump it later.
498     #
499     push(@HardLinks, $hdr);
500     $SpecialCnt++;
501     } else {
502     #
503     # Have to dump the original file. Just call the top-level
504     # routine, so that we save the hassle of dealing with
505     # mangling, merging and attributes.
506     #
507     $HardLinkExtraFiles{$hdr->{linkname}} = 1;
508     archiveWrite($fh, $hdr->{linkname}, $hdr->{name});
509     }
510     } elsif ( $hdr->{type} == BPC_FTYPE_SYMLINK ) {
511     #
512     # Symbolic link: read the symbolic link contents into the header
513     # and write the header.
514     #
515     my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0, $hdr->{compress});
516     if ( !defined($f) ) {
517     print(STDERR "Unable to open symlink file $hdr->{fullPath}\n");
518     $ErrorCnt++;
519     return;
520     }
521     my $data;
522     while ( $f->read(\$data, $BufSize) > 0 ) {
523     $hdr->{linkname} .= $data;
524     }
525     $f->close;
526     $hdr->{size} = 0;
527     TarWriteFileInfo($fh, $hdr);
528     $SpecialCnt++;
529     } elsif ( $hdr->{type} == BPC_FTYPE_CHARDEV
530     || $hdr->{type} == BPC_FTYPE_BLOCKDEV
531     || $hdr->{type} == BPC_FTYPE_FIFO ) {
532     #
533     # Special files: for char and block special we read the
534     # major and minor numbers from a plain file.
535     #
536     if ( $hdr->{type} != BPC_FTYPE_FIFO ) {
537     my $f = BackupPC::FileZIO->open($hdr->{fullPath}, 0,
538     $hdr->{compress});
539     my $data;
540     if ( !defined($f) || $f->read(\$data, $BufSize) < 0 ) {
541     print(STDERR "Unable to open/read char/block special file"
542     . " $hdr->{fullPath}\n");
543     $f->close if ( defined($f) );
544     $ErrorCnt++;
545     return;
546     }
547     $f->close;
548     if ( $data =~ /(\d+),(\d+)/ ) {
549     $hdr->{devmajor} = $1;
550     $hdr->{devminor} = $2;
551     }
552     }
553     $hdr->{size} = 0;
554     TarWriteFileInfo($fh, $hdr);
555     $SpecialCnt++;
556     } else {
557     print(STDERR "Got unknown type $hdr->{type} for $hdr->{name}\n");
558     $ErrorCnt++;
559     }
560     }
561 dpavlin 112
562     my $t_fmt = '%Y-%m-%d %H:%M:%S';
563     sub curr_time {
564     return strftime($t_fmt,localtime());
565     }

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26