/[psinib]/psinib.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /psinib.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.3 by dpavlin, Sat Jan 4 13:29:12 2003 UTC revision 1.11 by dpavlin, Sun Oct 12 15:58:28 2003 UTC
# Line 12  Line 12 
12  #  #
13  #  #
14  # usage:  # usage:
15  #       $ backup.pl mountscript  #       $ psinib.pl mountscript
16    
17  use strict 'vars';  use strict 'vars';
18  use Data::Dumper;  use Data::Dumper;
# Line 22  use List::Compare; Line 22  use List::Compare;
22  use Filesys::SmbClient;  use Filesys::SmbClient;
23  #use Taint;  #use Taint;
24  use Fcntl qw(LOCK_EX LOCK_NB);  use Fcntl qw(LOCK_EX LOCK_NB);
25    use Digest::MD5;
26    use File::Basename;
27    
28  # configuration  # configuration
29  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile
30  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir
31    
32  my $LOG = '/var/log/backup.log';        # add path here...  my $LOG = '/var/log/backup.log';        # add path here...
33  $LOG = '/tmp/backup.log';  #$LOG = '/tmp/backup.log';
34    
35  # store backups in which directory  # store backups in which directory
36  my $BACKUP_DEST = '/data/isis_backup';  my $BACKUP_DEST = '/backup/isis_backup';
37    
38  # files to ignore in backup  # files to ignore in backup
39  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');
40    
41  # open log  # open log
42  open(L, "> $LOG") || die "can't open log $LOG: $!";  open(L, ">> $LOG") || die "can't open log $LOG: $!";
43  select((select(L), $|=1)[0]);   # flush output  select((select(L), $|=1)[0]);   # flush output
44    
45  # make a lock on logfile  # make a lock on logfile
# Line 62  my $mounts = shift @ARGV || Line 64  my $mounts = shift @ARGV ||
64    
65  my @in_backup;  # shares which are backeduped this run  my @in_backup;  # shares which are backeduped this run
66    
67  my $p = new Net::Ping->new();  my $p = new Net::Ping->new("tcp", 2);
68    # ping will try tcp connect to netbios-ssn (139)
69    $p->{port_num} = getservbyname("netbios-ssn", "tcp");
70    
71  my $backup_ok = 0;  my $backup_ok = 0;
72    
73  my $smb;  my $smb;
74  my %smb_atime;  my %smb_atime;
75  my %smb_mtime;  my %smb_mtime;
76    my %file_md5;
77    
78  open(M, $mounts) || die "can't open $mounts: $!";  open(M, $mounts) || die "can't open $mounts: $!";
79  while(<M>) {  while(<M>) {
# Line 76  while(<M>) { Line 81  while(<M>) {
81          next if !/^\s*smbmount\s/;          next if !/^\s*smbmount\s/;
82          my (undef,$share,undef,$opt) = split(/\s+/,$_,4);          my (undef,$share,undef,$opt) = split(/\s+/,$_,4);
83    
84          my ($user,$passwd,$workgroup);          my ($user,$passwd,$workgroup,$ip);
85    
86          foreach (split(/,/,$opt)) {          foreach (split(/,/,$opt)) {
87                  my ($n,$v) = split(/=/,$_,2);                  my ($n,$v) = split(/=/,$_,2);
# Line 92  while(<M>) { Line 97  while(<M>) {
97                          }                          }
98                  } elsif ($n =~ m#workgroup#i) {                  } elsif ($n =~ m#workgroup#i) {
99                          $workgroup = $v;                          $workgroup = $v;
100                    } elsif ($n =~ m#ip#i) {
101                            $ip = $v;
102                  }                  }
103          }          }
104    
105          push @in_backup,$share;          push @in_backup,$share;
106    
107    
108            my ($host,$dir,$date_dir) = share2host_dir($share);
109            my $bl = "$BACKUP_DEST/$host/$dir/latest";      # latest backup
110            my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";   # current one
111            my $real_bl;
112            if (-l $bl) {
113                    $real_bl=readlink($bl) || die "can't read link $bl: $!";
114                    $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
115                    if (-l $bc && $real_bl eq $bc) {
116                            print "$share allready backuped...\n";
117                            $backup_ok++;
118                            next;
119                    }
120    
121            }
122    
123    
124          print "working on $share\n";          print "working on $share\n";
125    
126          my $ip = get_ip($share);          # try to nmblookup IP
127            $ip = get_ip($share) if (! $ip);
128    
129          if ($ip) {          if ($ip) {
130                  xlog($share,"IP is $ip");                  xlog($share,"IP is $ip");
# Line 117  xlog("","$backup_ok backups completed of Line 142  xlog("","$backup_ok backups completed of
142    
143  #-------------------------------------------------------------------------  #-------------------------------------------------------------------------
144    
145    
146  # get IP number from share  # get IP number from share
147  sub get_ip {  sub get_ip {
148          my $share = shift;          my $share = shift;
# Line 129  sub get_ip { Line 155  sub get_ip {
155          }          }
156  }  }
157    
158    
159    # write entry to screen and log
160  sub xlog {  sub xlog {
161          my $share = shift;          my $share = shift;
162          my $t = strftime $LOG_TIME_FMT, localtime;          my $t = strftime $LOG_TIME_FMT, localtime;
# Line 137  sub xlog { Line 165  sub xlog {
165          print L "$t $share\t$m\n";          print L "$t $share\t$m\n";
166  }  }
167    
168  sub snap_share {  # dump warn and dies into log
169    BEGIN { $SIG{'__WARN__'} = sub { xlog('WARN',$_[0]) ; warn $_[0] } }
170          my $share = shift;  BEGIN { $SIG{'__DIE__'} = sub { xlog('DIE',$_[0]) ; die $_[0] } }
   
         my %param = ( debug => 0 );  
171    
         $param{username} = shift;  
         $param{password} = shift;  
         $param{workgroup} = shift;  
172    
173    # split share name to host, dir and currnet date dir
174    sub share2host_dir {
175            my $share = shift;
176          my ($host,$dir);          my ($host,$dir);
177          if ($share =~ m#//([^/]+)/(.+)$#) {          if ($share =~ m#//([^/]+)/(.+)$#) {
178                  ($host,$dir) = ($1,$2);                  ($host,$dir) = ($1,$2);
# Line 157  sub snap_share { Line 183  sub snap_share {
183                  print "Can't parse share $share into host and directory!\n";                  print "Can't parse share $share into host and directory!\n";
184                  return;                  return;
185          }          }
186            return ($host,$dir,strftime $DIR_TIME_FMT, localtime);
187    }
188    
189    
190    # make a snapshot of a share
191    sub snap_share {
192    
193            my $share = shift;
194    
195          my $date_dir = strftime $DIR_TIME_FMT, localtime;          my %param = ( debug => 0 );
196    
197            $param{username} = shift || warn "can't find username for share $share";
198            $param{password} = shift || warn "can't find passwod for share $share";
199            $param{workgroup} = shift || warn "can't find workgroup for share $share";
200    
201            my ($host,$dir,$date_dir) = share2host_dir($share);
202    
203          # latest backup directory          # latest backup directory
204          my $bl = "$BACKUP_DEST/$host/$dir/latest";          my $bl = "$BACKUP_DEST/$host/$dir/latest";
# Line 166  sub snap_share { Line 206  sub snap_share {
206          my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";          my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";
207    
208          my $real_bl;          my $real_bl;
209          if (-e $bl) {          if (-l $bl) {
210                  $real_bl=readlink($bl) || die "can't read link $bl: $!";                  $real_bl=readlink($bl) || die "can't read link $bl: $!";
211                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
212          } else {          } else {
213                  print "no old backup, this is first run...\n";                  print "no old backup, trying to find last backup, ";
214                    if (opendir(BL_DIR, "$BACKUP_DEST/$host/$dir")) {
215                            my @bl_dirs = sort grep { !/^\./ && -d "$BACKUP_DEST/$host/$dir/$_" } readdir(BL_DIR);
216                            closedir(BL_DIR);
217                            $real_bl=pop @bl_dirs;
218                            print "using $real_bl as latest...\n";
219                            $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
220                            if ($real_bl eq $bc) {
221                                    xlog($share,"latest from today (possible partial backup)");
222                                    rename $real_bl,$real_bl.".partial" || warn "can't reaname partial backup: $!";
223                                    $real_bl .= ".partial";
224                            }
225                    } else {
226                            print "this is first run...\n";
227                    }
228          }          }
229    
230          if (-e $bc && $real_bl && $real_bl eq $bc) {          if (-l $bc && $real_bl && $real_bl eq $bc) {
231                  print "$share allready backuped...\n";                  print "$share allready backuped...\n";
232                  return;                  return;
233          }          }
# Line 202  sub snap_share { Line 256  sub snap_share {
256          my %file_size;          my %file_size;
257          my %file_atime;          my %file_atime;
258          my %file_mtime;          my %file_mtime;
259          my %file_md5;          #my %file_md5;
260    
261          my @smb_files;          my @smb_files;
262          my %smb_size;          my %smb_size;
263          #my %smb_atime;          #my %smb_atime;
264          #my %smb_mtime;          #my %smb_mtime;
         my %smb_md5;  
   
265    
266          sub norm_dir {          sub norm_dir {
267                  my $foo = shift;                  my $foo = shift;
# Line 225  sub snap_share { Line 277  sub snap_share {
277          my $di = 0;          my $di = 0;
278          while ($di <= $#dirs && $real_bl) {          while ($di <= $#dirs && $real_bl) {
279                  my $d=$dirs[$di++];                  my $d=$dirs[$di++];
280                  opendir(DIR,"$bl/$d") || warn "opendir($bl/$d): $!\n";                  opendir(DIR,"$real_bl/$d") || warn "opendir($real_bl/$d): $!\n";
281    
282                  # read .backupignore if exists                  # read .backupignore if exists
283                  if (-f "$bl/$d/.backupignore") {                  if (-f "$real_bl/$d/.backupignore") {
284                          open(I,"$bl/$d/.backupignore");                          open(I,"$real_bl/$d/.backupignore");
285                          while(<I>) {                          while(<I>) {
286                                  chomp;                                  chomp;
287                                  push @ignore,norm_dir("$d/$_");                                  push @ignore,norm_dir("$d/$_");
288                          }                          }
289                          close(I);                          close(I);
290  print STDERR "ignore: ",join("|",@ignore),"\n";  #print STDERR "ignore: ",join("|",@ignore),"\n";
291                          link "$bl/$d/.backupignore","$bc/$d/.backupignore" ||                          link "$real_bl/$d/.backupignore","$bc/$d/.backupignore" ||
292                                  warn "can't copy $bl/$d/.backupignore to current backup dir: $!\n";                                  warn "can't copy $real_bl/$d/.backupignore to current backup dir: $!\n";
293                  }                  }
294    
295                  # read .md5sum if exists                  # read .md5sum if exists
296                  if (-f "$bl/$d/.md5sum") {                  if (-f "$real_bl/$d/.md5sum") {
297                          open(I,"$bl/$d/.md5sum");                          open(I,"$real_bl/$d/.md5sum");
298                          while(<I>) {                          while(<I>) {
299                                  chomp;                                  chomp;
300                                  my ($md5,$f) = split(/\s+/,$_,2);                                  my ($md5,$f) = split(/\s+/,$_,2);
# Line 256  print STDERR "ignore: ",join("|",@ignore Line 308  print STDERR "ignore: ",join("|",@ignore
308                          next if ($f eq '.');                          next if ($f eq '.');
309                          next if ($f eq '..');                          next if ($f eq '..');
310                          my $pr = norm_dir("$d/$f");     # path relative                          my $pr = norm_dir("$d/$f");     # path relative
311                          my $pf = norm_dir("$d/$f","$bl/");      # path full                          my $pf = norm_dir("$d/$f","$real_bl/"); # path full
312                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {
313                                  if (-f $pf) {                                  if (-f $pf) {
314                                          push @files,$pr;                                          push @files,$pr;
# Line 285  print STDERR "ignore: ",join("|",@ignore Line 337  print STDERR "ignore: ",join("|",@ignore
337    
338          $di = 0;          $di = 0;
339          while ($di <= $#smb_dirs) {          while ($di <= $#smb_dirs) {
340                  my $d=$smb_dirs[$di++];                  my $d=$smb_dirs[$di];
341                  my $pf = norm_dir($d,"smb:$share/");    # path full                  my $pf = norm_dir($d,"smb:$share/");    # path full
342                  my $D = $smb->opendir($pf) || warn "smb->opendir($pf): $!\n";                  my $D = $smb->opendir($pf);
343                    if (! $D) {
344                            xlog($share,"FATAL: $share: $!");
345                            # remove failing dir
346                            delete $smb_dirs[$di];
347                            next;
348                    }
349                    $di++;
350    
351                  my @clutter = $smb->readdir_struct($D);                  my @clutter = $smb->readdir_struct($D);
352                  foreach my $item (@clutter) {                  foreach my $item (@clutter) {
# Line 350  print STDERR "ignore: ",join("|",@ignore Line 409  print STDERR "ignore: ",join("|",@ignore
409                                  next;                                  next;
410                          }                          }
411    
412                            my $md5 = Digest::MD5->new;
413    
414                          my $fd = $smb->open("$from/$f");                          my $fd = $smb->open("$from/$f");
415                          if (! $fd) {                          if (! $fd) {
416                                  print STDERR "can't open smb file $from/$f: $!\n";                                  print STDERR "can't open smb file $from/$f: $!\n";
# Line 359  print STDERR "ignore: ",join("|",@ignore Line 420  print STDERR "ignore: ",join("|",@ignore
420                          while (defined(my $b=$smb->read($fd,4096))) {                          while (defined(my $b=$smb->read($fd,4096))) {
421                                  print F $b;                                  print F $b;
422                                  $l += length($b);                                  $l += length($b);
423                                    $md5->add($b);
424                          }                          }
425    
426                          $smb->close($fd);                          $smb->close($fd);
427                          close(F);                          close(F);
428    
429                            $file_md5{$f} = $md5->hexdigest;
430    
431                          # FIX: this fails with -T                          # FIX: this fails with -T
432                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];
433                          utime $a, $m, "$to/$f" ||                          utime $a, $m, "$to/$f" ||
# Line 417  print STDERR "ignore: ",join("|",@ignore Line 481  print STDERR "ignore: ",join("|",@ignore
481          xlog($share,"$transfer bytes transfered...");          xlog($share,"$transfer bytes transfered...");
482    
483          foreach (@ln_files) {          foreach (@ln_files) {
484                  link "$bl/$_","$bc/$_" || warn "link $bl/$_ -> $bc/$_: $!\n";                  link "$real_bl/$_","$bc/$_" || warn "link $real_bl/$_ -> $bc/$_: $!\n";
485          }          }
486    
487          # remove files          # remove files
# Line 430  print STDERR "ignore: ",join("|",@ignore Line 494  print STDERR "ignore: ",join("|",@ignore
494                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";
495          }          }
496    
497            # remove old .md5sum
498          # FIX: create .md5sum          foreach (sort @dirs) {
499                    unlink "$bc/$_/.md5sum" if (-e "$bc/$_/.md5sum");
500            }
501    
502            # create .md5sum
503            my $last_dir = '';
504            my $md5;
505            foreach my $f (sort { $file_md5{$a} cmp $file_md5{$b} } keys %file_md5) {
506                    my $dir = dirname($f);
507                    my $file = basename($f);
508    #print "$f -- $dir / $file<--\n";
509                    if ($dir ne $last_dir) {
510                            close($md5) if ($md5);
511                            open($md5, ">> $bc/$dir/.md5sum") || warn "can't create $bc/$dir/.md5sum: $!";
512                            $last_dir = $dir;
513    #print STDERR "writing $last_dir/.md5sum\n";
514                    }
515                    print $md5 $file_md5{$f},"  $file\n";
516            }
517            close($md5) if ($md5);
518    
519          # create leatest link          # create leatest link
520    #print "ln -s $bc $real_bl\n";
521            if (-l $bl) {
522                    unlink $bl || warn "can't remove old latest symlink $bl: $!\n";
523            }
524          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";
525    
526            # FIX: sanity check -- remove for speedup
527            xlog($share,"failed to create latest symlink $bl -> $bc...") if (readlink($bl) ne $bc || ! -l $bl);
528    
529          xlog($share,"backup completed...");          xlog($share,"backup completed...");
530  }  }
531    
# Line 493  transferring whole file over network (ju Line 583  transferring whole file over network (ju
583  over network)  over network)
584    
585  =item - usage of C<.md5sum> files (compatible with command-line utility  =item - usage of C<.md5sum> files (compatible with command-line utility
586  C<md5sum> to keep file between snapshots hard-linked  C<md5sum>) to keep file between snapshots hard-linked
587    
588  =back  =back
589    
# Line 501  C<md5sum> to keep file between snapshots Line 591  C<md5sum> to keep file between snapshots
591    
592  This section is not yet written.  This section is not yet written.
593    
594  =head1 BUGS and LIMITATIONS  =head1 HACKS, TRICKS, BUGS and LIMITATIONS
595    
596    This chapter will have all content that doesn't fit anywhere else.
597    
598    =head2 Can snapshots be more frequent than daily?
599    
600  There is not real reason why you can't take snapshot more often than  There is not real reason why you can't take snapshot more often than
601  one a day. Actually, if you are using B<psinib> to backup Windows workstations  once a day. Actually, if you are using B<psinib> to backup Windows
602  they tend to come-and-go, so running B<psinib> several times a day  workstations you already know that they tend to come-and-go during the day
603  increases your chance of having up-to-date backup (B<psinib> will not  (reboots probably ;-), so running B<psinib> several times a day increases
604  make multiple backups for same day if such backup already exists).  your chance of having up-to-date backup (B<psinib> will not make multiple
605    snapshots for same day, nor will it update snapshot for current day if
606    it already exists).
607    
608  However, changing that to produce backups which are, for example, hourly  However, changing B<psinib> to produce snapshots which are, for example, hourly
609  is a simple change of C<$DIR_TIME_FMT> which is currently set to  is a simple change of C<$DIR_TIME_FMT> which is currently set to
610  C<'%Y%m%d'> (see I<strftime> documentation for explanation of that  C<'%Y%m%d'> (see I<strftime> documentation for explanation of that
611  format). If you change that to C<'%Y%m%d-%H> you can have hourly snapshots  format). If you change that to C<'%Y%m%d-%H> you can have hourly snapshots
# Line 517  format). If you change that to C<'%Y%m%d Line 613  format). If you change that to C<'%Y%m%d
613  program will sound strange, but other than that it should work.  program will sound strange, but other than that it should work.
614  I<You have been warned>.  I<You have been warned>.
615    
616    =head2 Do I really need to share every directory which I want to snapshot?
617    
618    Actually, no. Due to usage of C<Filesys::SmbClient> module, you can also
619    specify sub-directory inside your share that you want to backup. This feature
620    is most useful if you want to use administrative shares (but, have in mind
621    that you have to enter your Win administrator password in unencrypted file on
622    disk to do that) like this:
623    
624            smbmount //server/c$/WinNT/fonts  /mnt  -o username=administrator%win  
625    
626    After that you will get directories with snapshots like:
627    
628            server/c_WinNT_fonts/yyyymmdd/....
629    
630    =head2 Won't I run out of disk space?
631    
632    Of course you will... Snapshots and logfiles will eventually fill-up your disk.
633    However, you can do two things to stop that:
634    
635    =head3 Clean snapshort older than x days
636    
637    You can add following command to your C<root> crontab:
638    
639            find /backup/isis_backup -type d -mindepth 3 -maxdepth 3 -mtime +11 -exec rm -Rf {} \;
640    
641    I assume that C</backup/isis_backup> is directory in which are your snapshots
642    and that you don't want to keep snapshots older than 11 days (that's
643    C<-mtime +11> part of command).
644    
645    =head3 Rotate your logs
646    
647    I will leave that to you. I relay on GNU/Debian's C<logrotate> to do it for me.
648    
649    =head2 What are I<YYYYMMDD.partial> directories?
650    
651    If there isn't I<latest> symlink in snapshot directory, it's preatty safe to
652    assume that previous backup from that day failed. So, that directory will
653    be renamed to I<YYYYMMDD.partial> and snapshot will be performed again,
654    linking same files (other alternative would be to erase that dir and find
655    second-oldest directory, but this seemed like more correct approach).
656    
657  =head1 AUTHOR  =head1 AUTHOR
658    
659  Dobrica Pavlinusic <dpavlin@rot13.org>  Dobrica Pavlinusic <dpavlin@rot13.org>
Legend:
Removed from v.1.3  
changed lines
  Added in v.1.11
  ViewVC Help
Powered by ViewVC 1.1.26