/[BackupPC]/trunk/bin/BackupPC_incPartsUpdate
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/bin/BackupPC_incPartsUpdate

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 271 by dpavlin, Tue Dec 13 18:29:05 2005 UTC revision 323 by dpavlin, Tue Jan 31 11:11:37 2006 UTC
# Line 16  use Archive::Tar::Streamed; Line 16  use Archive::Tar::Streamed;
16  use Algorithm::Diff;  use Algorithm::Diff;
17  use Getopt::Std;  use Getopt::Std;
18  use File::Slurp;  use File::Slurp;
19    use File::Pid;
20    
21    =head1 NAME
22    
23    BackupPC_incPartsUpdate
24    
25    =head1 DESCRIPTION
26    
27    Create C<.tar.gz> increments on disk calling C<BackupPC_tarIncCreate>.
28    
29    Following options are supported (but all are optional):
30    
31    =over 4
32    
33    =item -h hostname
34    
35    Update parts for just single C<hostname>
36    
37    =item -c
38    
39    Force check for tar archives which exist on disk
40    
41    =item -d
42    
43    Turn debugging output
44    
45    =back
46    
47    =cut
48    
49    my %opt;
50    getopts("cdh:", \%opt );
51    
52    my $debug = $opt{d};
53    my $check = $opt{c} && print STDERR "NOTICE: tar archive check forced\n";
54    
55    my $pid_path = abs_path($0);
56    $pid_path =~ s/\W+/_/g;
57    
58    my $pidfile = new File::Pid({
59            file => "/tmp/$pid_path",
60    });
61    
62    if (my $pid = $pidfile->running ) {
63            die "$0 already running: $pid\n";
64    } elsif ($pidfile->pid ne $$) {
65            $pidfile->remove;
66            $pidfile = new File::Pid;
67    }
68    
69    print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";
70    $pidfile->write;
71    
72  my $bpc = BackupPC::Lib->new || die "can't create BackupPC::Lib";  my $bpc = BackupPC::Lib->new || die "can't create BackupPC::Lib";
73  my %Conf = $bpc->Conf();  my %Conf = $bpc->Conf();
# Line 34  foreach my $c (qw/gzip md5sum/) { Line 86  foreach my $c (qw/gzip md5sum/) {
86          $bin->{$c} = which($c) || die "$0 needs $c, install it\n";          $bin->{$c} = which($c) || die "$0 needs $c, install it\n";
87  }  }
88    
 my %opt;  
 getopts("cd", \%opt );  
   
 my $debug = $opt{d};  
 my $check = $opt{c} && print STDERR "NOTICE: tar archive check forced\n";  
   
89  $|=1;  $|=1;
90    
91  my $start_t = time();  my $start_t = time();
# Line 84  sub get_backup_id($$$) { Line 130  sub get_backup_id($$$) {
130                  FROM backups                  FROM backups
131                  INNER JOIN shares       ON backups.shareID=shares.ID                  INNER JOIN shares       ON backups.shareID=shares.ID
132                  INNER JOIN hosts        ON backups.hostID = hosts.ID                  INNER JOIN hosts        ON backups.hostID = hosts.ID
133                  where hosts.name = ? and shares.name = ? and backups.num = ?                  WHERE hosts.name = ? and shares.name = ? and backups.num = ?
134          });          });
135          $sth->execute($host, $share, $num);          $sth->execute($host, $share, $num);
136          my ($id) = $sth->fetchrow_array;          my ($id) = $sth->fetchrow_array;
# Line 96  sub get_backup_id($$$) { Line 142  sub get_backup_id($$$) {
142          return $id;          return $id;
143  }  }
144    
145    sub backup_inc_deleted($) {
146            my $backup_id = shift;
147            my $sth_inc_deleted = $dbh->prepare(qq{
148                    update backups set
149                            inc_deleted = true,
150                            parts = 0
151                    where id = ?
152            });
153            $sth_inc_deleted->execute($backup_id);
154    }
155    
156  sub tar_check($$$$) {  sub tar_check($$$$) {
157          my ($host,$share,$num,$filename) = @_;          my ($host,$share,$num,$filename) = @_;
# Line 103  sub tar_check($$$$) { Line 159  sub tar_check($$$$) {
159          my $t = time();          my $t = time();
160          print curr_time, " check $host:$share#$num -> $filename";          print curr_time, " check $host:$share#$num -> $filename";
161    
162            # depending on expected returned value this is used like:
163            # my $uncompress_size = get_gzip_size('/full/path/to.gz');
164            # my ($compress_size, $uncompress_size) = get_gzip_size('/path.gz');
165            sub get_gzip_size($) {
166                    my $filename = shift;
167                    die "file $filename problem: $!" unless (-r $filename);
168                    open(my $gzip, $bin->{gzip}." -l $filename |") || die "can't gzip -l $filename: $!";
169                    my $line = <$gzip>;
170                    chomp($line);
171                    $line = <$gzip> if ($line =~ /^\s+compressed/);
172    
173                    my ($comp, $uncomp) = (0,0);
174    
175                    if ($line =~ m/^\s+(\d+)\s+(\d+)\s+\d+\.\d+/) {
176                            if (wantarray) {
177                                    return [ $1, $2 ];
178                            } else {
179                                    return $2;
180                            }
181                    } else {
182                            die "can't find size in line: $line";
183                    }
184            }
185    
186          sub check_part {          sub check_part {
187                  my ($host, $share, $num, $part_nr, $tar_size, $size, $md5, $items) = @_;                  my ($host, $share, $num, $part_nr, $tar_size, $size, $md5, $items) = @_;
188                  my $backup_id = get_backup_id($host, $share, $num);                  my $backup_id = get_backup_id($host, $share, $num);
# Line 165  sub tar_check($$$$) { Line 245  sub tar_check($$$$) {
245    
246                  print "\n\t- $tarfilename";                  print "\n\t- $tarfilename";
247    
248                  my $size = (stat( "$tar_dir/$tarfilename" ))[7] || die "can't stat $tar_dir/$tarfilename";                  my $path = "$tar_dir/$tarfilename";
249    
250                    my $size = (stat( $path ))[7] || die "can't stat $path: $!";
251    
252                  if ($size > $Conf{MaxArchiveSize}) {                  if ($size > $Conf{MaxArchiveSize}) {
253                          print STDERR " part bigger than media $size > $Conf{MaxArchiveSize} }}" if ($debug);                          print ", part bigger than media $size > $Conf{MaxArchiveSize}\n";
254                          $same = 0;                          return 0;
                         last;  
255                  }                  }
256    
257                  print ", $size bytes";                  print ", $size bytes";
258    
                 my $path = "$tar_dir/$tarfilename";  
259    
260                  open(my $fh, "gzip -cd $tar_dir/$tarfilename |") or die "can't open $tar_dir/$tarfilename: $!";                  open(my $fh, "gzip -cd $path |") or die "can't open $path: $!";
261                  binmode($fh);                  binmode($fh);
262                  my $tar = Archive::Tar::Streamed->new($fh);                  my $tar = Archive::Tar::Streamed->new($fh);
263    
264                  my $tar_size = 0;                  my $tar_size_inarc = 0;
265                  my $items = 0;                  my $items = 0;
266    
267                  while(my $entry = $tar->next) {                  while(my $entry = $tar->next) {
268                          push @tar_files, $entry->name;                          push @tar_files, $entry->name;
269                          $items++;                          $items++;
270                          $tar_size += $entry->size;                          $tar_size_inarc += $entry->size;
271    
272                          if ($tar_size > $Conf{MaxArchiveFileSize}) {                          if ($tar_size_inarc > $Conf{MaxArchiveFileSize}) {
273                                  print ", part $tarfilename is too big $tar_size > $Conf{MaxArchiveFileSize}\n";                                  print ", part $tarfilename is too big $tar_size_inarc > $Conf{MaxArchiveFileSize}\n";
274                                  $same = 0;                                  return 0;
                                 last;  
275                          }                          }
276    
277                  }                  }
278    
279                    close($fh);
280    
281                  print ", $items items";                  print ", $items items";
282    
283                    if ($tar_size_inarc == 0 && $items == 0) {
284                            print ", EMPTY tar\n";
285    
286                            my $backup_id = get_backup_id($host, $share, $num);
287                            backup_inc_deleted( $backup_id );
288    
289                            $dbh->commit;
290    
291                            return 1;
292                    }
293    
294                    my $tar_size = get_gzip_size( $path );
295    
296                    # real tar size is bigger because of padding    
297                    if ($tar_size_inarc > $tar_size) {
298                            print ", size of files in tar ($tar_size_inarc) bigger than whole tar ($tar_size)!\n";
299                            return 0;
300                    }
301    
302                  #                  #
303                  # check if md5 exists, and if not, create one                  # check if md5 exists, and if not, create one
304                  #                  #
# Line 282  select Line 382  select
382          hosts.name as host,          hosts.name as host,
383          shares.name as share,          shares.name as share,
384          backups.num as num,          backups.num as num,
385            backups.date,
386          inc_size,          inc_size,
387          parts          parts,
388            count(backup_parts.backup_id) as backup_parts
389  from backups  from backups
390          join shares on backups.hostid = shares.hostid          join shares on backups.hostid = shares.hostid
391                  and shares.id = backups.shareid                  and shares.id = backups.shareid
392          join hosts on shares.hostid = hosts.id          join hosts on shares.hostid = hosts.id
393  where not inc_deleted          full outer join backup_parts on backups.id = backup_parts.backup_id
394    where not inc_deleted and backups.size > 0
395    group by backups.id, hosts.name, shares.name, backups.num, backups.date, inc_size, parts, backup_parts.backup_id
396  order by backups.date  order by backups.date
397    
398  } );  } );
# Line 297  $sth->execute(); Line 401  $sth->execute();
401  my $num_backups = $sth->rows;  my $num_backups = $sth->rows;
402  my $curr_backup = 1;  my $curr_backup = 1;
403    
404    if ($opt{h}) {
405            warn "making increments just for host $opt{h}\n";
406    }
407    
408  while (my $row = $sth->fetchrow_hashref) {  while (my $row = $sth->fetchrow_hashref) {
409    
410            if ($opt{h} && $row->{host} ne $opt{h}) {
411                    warn "skipped $row->{host}\n" if ($debug);
412                    next;
413            }
414    
415          $curr_backup++;          $curr_backup++;
416    
417          my $tar_file = BackupPC::SearchLib::getGzipName($row->{'host'}, $row->{'share'}, $row->{'num'});          my $tar_file = BackupPC::SearchLib::getGzipName($row->{'host'}, $row->{'share'}, $row->{'num'});
# Line 308  while (my $row = $sth->fetchrow_hashref) Line 421  while (my $row = $sth->fetchrow_hashref)
421    
422          print "# size: $size backup.size: ", $row->{inc_size},"\n" if ($opt{d});          print "# size: $size backup.size: ", $row->{inc_size},"\n" if ($opt{d});
423    
424          if ( $row->{'inc_size'} != -1 && $size != -1 && $row->{'inc_size'} >= $size) {          if ( $row->{'inc_size'} != -1 && $size != -1 && $row->{'inc_size'} >= $size && $row->{parts} == $row->{backup_parts}) {
425                  if ($check) {                  if ($check) {
426                          tar_check($row->{'host'}, $row->{'share'}, $row->{'num'}, $tar_file) && next;                          tar_check($row->{'host'}, $row->{'share'}, $row->{'num'}, $tar_file) && next;
427                  } else {                  } else {
# Line 316  while (my $row = $sth->fetchrow_hashref) Line 429  while (my $row = $sth->fetchrow_hashref)
429                  }                  }
430          }          }
431    
432          print curr_time, " creating $curr_backup/$num_backups ", $row->{'host'}, ":", $row->{'share'}, " #", $row->{'num'}, " -> $tar_file";          print curr_time, " creating $curr_backup/$num_backups ", $row->{host}, ":", $row->{share}, " #", $row->{num},
433                    " ", strftime('%Y-%m-%d', localtime($row->{date})), " -> $tar_file";
434    
435          my $t = time();          my $t = time();
436    
437          # re-create archive?          # re-create archive?
438          my $cmd = qq{ $tarIncCreate -h "$row->{'host'}" -s "$row->{'share'}" -n $row->{'num'} -f };          my $cmd = qq[ $tarIncCreate -h "$row->{host}" -s "$row->{share}" -n $row->{num} -f ];
439          print STDERR "## $cmd\n" if ($debug);          print STDERR "## $cmd\n" if ($debug);
440    
441          if (system($cmd) != 0) {          if (system($cmd) != 0) {
442                  print STDERR " FAILED";                  print STDERR " FAILED, marking this backup deleted";
443                    backup_inc_deleted( $row->{backup_id} );
444          }          }
445    
446          print ", dur: ",fmt_time(time() - $t), "\n";          print ", dur: ",fmt_time(time() - $t), "\n";
Legend:
Removed from v.271  
changed lines
  Added in v.323
  ViewVC Help
Powered by ViewVC 1.1.26