/[psinib]/psinib.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /psinib.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.11 - (show annotations)
Sun Oct 12 15:58:28 2003 UTC (20 years, 5 months ago) by dpavlin
Branch: MAIN
Changes since 1.10: +6 -4 lines
File MIME type: text/plain
add support for ip=1.2.3.4 option which overrides nmblookup

1 #!/usr/bin/perl -w
2 #
3 # psinib - Perl Snapshot Is Not Incremental Backup
4 #
5 # written by Dobrica Pavlinusic <dpavlin@rot13.org> 2003-01-03
6 # released under GPL v2 or later.
7 #
8 # Backup SMB directories using file produced by LinNeighbourhood (or some
9 # other program [vi :-)] which produces file in format:
10 #
11 # smbmount service mountpoint options
12 #
13 #
14 # usage:
15 # $ psinib.pl mountscript
16
17 use strict 'vars';
18 use Data::Dumper;
19 use Net::Ping;
20 use POSIX qw(strftime);
21 use List::Compare;
22 use Filesys::SmbClient;
23 #use Taint;
24 use Fcntl qw(LOCK_EX LOCK_NB);
25 use Digest::MD5;
26 use File::Basename;
27
28 # configuration
29 my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile
30 my $DIR_TIME_FMT = '%Y%m%d'; # strftime format for backup dir
31
32 my $LOG = '/var/log/backup.log'; # add path here...
33 #$LOG = '/tmp/backup.log';
34
35 # store backups in which directory
36 my $BACKUP_DEST = '/backup/isis_backup';
37
38 # files to ignore in backup
39 my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');
40
41 # open log
42 open(L, ">> $LOG") || die "can't open log $LOG: $!";
43 select((select(L), $|=1)[0]); # flush output
44
45 # make a lock on logfile
46
47 my $c = 0;
48 {
49 flock L, LOCK_EX | LOCK_NB and last;
50 sleep 1;
51 redo if ++$c < 10;
52 # no response for 10 sec, bail out
53 print STDERR "can't take lock on $LOG -- another $0 running?\n";
54 exit 1;
55 }
56
57 # taint path: nmblookup should be there!
58 $ENV{'PATH'} = "/usr/bin:/bin";
59
60 my $mounts = shift @ARGV ||
61 'mountscript';
62 # die "usage: $0 mountscript";
63
64
65 my @in_backup; # shares which are backeduped this run
66
67 my $p = new Net::Ping->new("tcp", 2);
68 # ping will try tcp connect to netbios-ssn (139)
69 $p->{port_num} = getservbyname("netbios-ssn", "tcp");
70
71 my $backup_ok = 0;
72
73 my $smb;
74 my %smb_atime;
75 my %smb_mtime;
76 my %file_md5;
77
78 open(M, $mounts) || die "can't open $mounts: $!";
79 while(<M>) {
80 chomp;
81 next if !/^\s*smbmount\s/;
82 my (undef,$share,undef,$opt) = split(/\s+/,$_,4);
83
84 my ($user,$passwd,$workgroup,$ip);
85
86 foreach (split(/,/,$opt)) {
87 my ($n,$v) = split(/=/,$_,2);
88 if ($n =~ m/username/i) {
89 if ($v =~ m#^(.+)/(.+)%(.+)$#) {
90 ($user,$passwd,$workgroup) = ($1,$2,$3);
91 } elsif ($v =~ m#^(.+)/(.+)$#) {
92 ($user,$workgroup) = ($1,$2);
93 } elsif ($v =~ m#^(.+)%(.+)$#) {
94 ($user,$passwd) = ($1,$2);
95 } else {
96 $user = $v;
97 }
98 } elsif ($n =~ m#workgroup#i) {
99 $workgroup = $v;
100 } elsif ($n =~ m#ip#i) {
101 $ip = $v;
102 }
103 }
104
105 push @in_backup,$share;
106
107
108 my ($host,$dir,$date_dir) = share2host_dir($share);
109 my $bl = "$BACKUP_DEST/$host/$dir/latest"; # latest backup
110 my $bc = "$BACKUP_DEST/$host/$dir/$date_dir"; # current one
111 my $real_bl;
112 if (-l $bl) {
113 $real_bl=readlink($bl) || die "can't read link $bl: $!";
114 $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
115 if (-l $bc && $real_bl eq $bc) {
116 print "$share allready backuped...\n";
117 $backup_ok++;
118 next;
119 }
120
121 }
122
123
124 print "working on $share\n";
125
126 # try to nmblookup IP
127 $ip = get_ip($share) if (! $ip);
128
129 if ($ip) {
130 xlog($share,"IP is $ip");
131 if ($p->ping($ip)) {
132 snap_share($share,$user,$passwd,$workgroup);
133 $backup_ok++;
134 }
135 }
136 }
137 close(M);
138
139 xlog("","$backup_ok backups completed of total ".($#in_backup+1)." this time (".int($backup_ok*100/($#in_backup+1))." %)");
140
141 1;
142
143 #-------------------------------------------------------------------------
144
145
146 # get IP number from share
147 sub get_ip {
148 my $share = shift;
149
150 my $host = $1 if ($share =~ m#//([^/]+)/#);
151
152 my $ip = `nmblookup $host`;
153 if ($ip =~ m/(\d+\.\d+\.\d+\.\d+)\s$host/i) {
154 return $1;
155 }
156 }
157
158
159 # write entry to screen and log
160 sub xlog {
161 my $share = shift;
162 my $t = strftime $LOG_TIME_FMT, localtime;
163 my $m = shift || '[no log entry]';
164 print STDERR $m,"\n";
165 print L "$t $share\t$m\n";
166 }
167
168 # dump warn and dies into log
169 BEGIN { $SIG{'__WARN__'} = sub { xlog('WARN',$_[0]) ; warn $_[0] } }
170 BEGIN { $SIG{'__DIE__'} = sub { xlog('DIE',$_[0]) ; die $_[0] } }
171
172
173 # split share name to host, dir and currnet date dir
174 sub share2host_dir {
175 my $share = shift;
176 my ($host,$dir);
177 if ($share =~ m#//([^/]+)/(.+)$#) {
178 ($host,$dir) = ($1,$2);
179 $dir =~ s/\W/_/g;
180 $dir =~ s/^_+//;
181 $dir =~ s/_+$//;
182 } else {
183 print "Can't parse share $share into host and directory!\n";
184 return;
185 }
186 return ($host,$dir,strftime $DIR_TIME_FMT, localtime);
187 }
188
189
190 # make a snapshot of a share
191 sub snap_share {
192
193 my $share = shift;
194
195 my %param = ( debug => 0 );
196
197 $param{username} = shift || warn "can't find username for share $share";
198 $param{password} = shift || warn "can't find passwod for share $share";
199 $param{workgroup} = shift || warn "can't find workgroup for share $share";
200
201 my ($host,$dir,$date_dir) = share2host_dir($share);
202
203 # latest backup directory
204 my $bl = "$BACKUP_DEST/$host/$dir/latest";
205 # current backup directory
206 my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";
207
208 my $real_bl;
209 if (-l $bl) {
210 $real_bl=readlink($bl) || die "can't read link $bl: $!";
211 $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
212 } else {
213 print "no old backup, trying to find last backup, ";
214 if (opendir(BL_DIR, "$BACKUP_DEST/$host/$dir")) {
215 my @bl_dirs = sort grep { !/^\./ && -d "$BACKUP_DEST/$host/$dir/$_" } readdir(BL_DIR);
216 closedir(BL_DIR);
217 $real_bl=pop @bl_dirs;
218 print "using $real_bl as latest...\n";
219 $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
220 if ($real_bl eq $bc) {
221 xlog($share,"latest from today (possible partial backup)");
222 rename $real_bl,$real_bl.".partial" || warn "can't reaname partial backup: $!";
223 $real_bl .= ".partial";
224 }
225 } else {
226 print "this is first run...\n";
227 }
228 }
229
230 if (-l $bc && $real_bl && $real_bl eq $bc) {
231 print "$share allready backuped...\n";
232 return;
233 }
234
235 die "You should really create BACKUP_DEST [$BACKUP_DEST] by hand! " if (!-e $BACKUP_DEST);
236
237 if (! -e "$BACKUP_DEST/$host") {
238 mkdir "$BACKUP_DEST/$host" || die "can't make dir for host $host, $BACKUP_DEST/$host: $!";
239 print "created host directory $BACKUP_DEST/$host...\n";
240 }
241
242 if (! -e "$BACKUP_DEST/$host/$dir") {
243 mkdir "$BACKUP_DEST/$host/$dir" || die "can't make dir for share $share, $BACKUP_DEST/$host/$dir $!";
244 print "created dir for share $share, $BACKUP_DEST/$host/$dir...\n";
245 }
246
247 mkdir $bc || die "can't make dir for current backup $bc: $!";
248
249 my @dirs = ( "/" );
250 my @smb_dirs = ( "/" );
251
252 my $transfer = 0; # bytes transfered over network
253
254 # this will store all available files and sizes
255 my @files;
256 my %file_size;
257 my %file_atime;
258 my %file_mtime;
259 #my %file_md5;
260
261 my @smb_files;
262 my %smb_size;
263 #my %smb_atime;
264 #my %smb_mtime;
265
266 sub norm_dir {
267 my $foo = shift;
268 my $prefix = shift;
269 $foo =~ s#//+#/#g;
270 $foo =~ s#/+$##g;
271 $foo =~ s#^/+##g;
272 return $prefix.$foo if ($prefix);
273 return $foo;
274 }
275
276 # read local filesystem
277 my $di = 0;
278 while ($di <= $#dirs && $real_bl) {
279 my $d=$dirs[$di++];
280 opendir(DIR,"$real_bl/$d") || warn "opendir($real_bl/$d): $!\n";
281
282 # read .backupignore if exists
283 if (-f "$real_bl/$d/.backupignore") {
284 open(I,"$real_bl/$d/.backupignore");
285 while(<I>) {
286 chomp;
287 push @ignore,norm_dir("$d/$_");
288 }
289 close(I);
290 #print STDERR "ignore: ",join("|",@ignore),"\n";
291 link "$real_bl/$d/.backupignore","$bc/$d/.backupignore" ||
292 warn "can't copy $real_bl/$d/.backupignore to current backup dir: $!\n";
293 }
294
295 # read .md5sum if exists
296 if (-f "$real_bl/$d/.md5sum") {
297 open(I,"$real_bl/$d/.md5sum");
298 while(<I>) {
299 chomp;
300 my ($md5,$f) = split(/\s+/,$_,2);
301 $file_md5{$f}=$md5;
302 }
303 close(I);
304 }
305
306 my @clutter = readdir(DIR);
307 foreach my $f (@clutter) {
308 next if ($f eq '.');
309 next if ($f eq '..');
310 my $pr = norm_dir("$d/$f"); # path relative
311 my $pf = norm_dir("$d/$f","$real_bl/"); # path full
312 if (grep(/^\Q$pr\E$/,@ignore) == 0) {
313 if (-f $pf) {
314 push @files,$pr;
315 $file_size{$pr}=(stat($pf))[7];
316 $file_atime{$pr}=(stat($pf))[8];
317 $file_mtime{$pr}=(stat($pf))[9];
318 } elsif (-d $pf) {
319 push @dirs,$pr;
320 } else {
321 print STDERR "unknown type: $pf\n";
322 }
323 } else {
324 print STDERR "ignored: $pr\n";
325 }
326 }
327 }
328
329 xlog($share,($#files+1)." files and ".($#dirs+1)." dirs on local disk before backup");
330
331 # read smb filesystem
332
333 xlog($share,"smb to $share as $param{username}/$param{workgroup}");
334
335 # FIX: how to aviod creation of ~/.smb/smb.conf ?
336 $smb = new Filesys::SmbClient(%param) || die "SmbClient :$!\n";
337
338 $di = 0;
339 while ($di <= $#smb_dirs) {
340 my $d=$smb_dirs[$di];
341 my $pf = norm_dir($d,"smb:$share/"); # path full
342 my $D = $smb->opendir($pf);
343 if (! $D) {
344 xlog($share,"FATAL: $share: $!");
345 # remove failing dir
346 delete $smb_dirs[$di];
347 next;
348 }
349 $di++;
350
351 my @clutter = $smb->readdir_struct($D);
352 foreach my $item (@clutter) {
353 my $f = $item->[1];
354 next if ($f eq '.');
355 next if ($f eq '..');
356 my $pr = norm_dir("$d/$f"); # path relative
357 my $pf = norm_dir("$d/$f","smb:$share/"); # path full
358 if (grep(/^\Q$pr\E$/,@ignore) == 0) {
359 if ($item->[0] == main::SMBC_FILE) {
360 push @smb_files,$pr;
361 $smb_size{$pr}=($smb->stat($pf))[7];
362 $smb_atime{$pr}=($smb->stat($pf))[10];
363 $smb_mtime{$pr}=($smb->stat($pf))[11];
364 } elsif ($item->[0] == main::SMBC_DIR) {
365 push @smb_dirs,$pr;
366 } else {
367 print STDERR "unknown type: $pf\n";
368 }
369 } else {
370 print STDERR "smb ignored: $pr\n";
371 }
372 }
373 }
374
375 xlog($share,($#smb_files+1)." files and ".($#smb_dirs+1)." dirs on remote share");
376
377 # sync dirs
378 my $lc = List::Compare->new(\@dirs, \@smb_dirs);
379
380 my @dirs2erase = $lc->get_Lonly;
381 my @dirs2create = $lc->get_Ronly;
382 xlog($share,($#dirs2erase+1)." dirs to erase and ".($#dirs2create+1)." dirs to create");
383
384 # create new dirs
385 foreach (sort @smb_dirs) {
386 mkdir "$bc/$_" || warn "mkdir $_: $!\n";
387 }
388
389 # sync files
390 $lc = List::Compare->new(\@files, \@smb_files);
391
392 my @files2erase = $lc->get_Lonly;
393 my @files2create = $lc->get_Ronly;
394 xlog($share,($#files2erase+1)." files to erase and ".($#files2create+1)." files to create");
395
396 sub smb_copy {
397 my $smb = shift;
398
399 my $from = shift;
400 my $to = shift;
401
402
403 my $l = 0;
404
405 foreach my $f (@_) {
406 #print "smb_copy $from/$f -> $to/$f\n";
407 if (! open(F,"> $to/$f")) {
408 print STDERR "can't open new file $to/$f: $!\n";
409 next;
410 }
411
412 my $md5 = Digest::MD5->new;
413
414 my $fd = $smb->open("$from/$f");
415 if (! $fd) {
416 print STDERR "can't open smb file $from/$f: $!\n";
417 next;
418 }
419
420 while (defined(my $b=$smb->read($fd,4096))) {
421 print F $b;
422 $l += length($b);
423 $md5->add($b);
424 }
425
426 $smb->close($fd);
427 close(F);
428
429 $file_md5{$f} = $md5->hexdigest;
430
431 # FIX: this fails with -T
432 my ($a,$m) = ($smb->stat("$from/$f"))[10,11];
433 utime $a, $m, "$to/$f" ||
434 warn "can't update utime on $to/$f: $!\n";
435
436 }
437 return $l;
438 }
439
440 # copy new files
441 foreach (@files2create) {
442 $transfer += smb_copy($smb,"smb:$share",$bc,$_);
443 }
444
445 my $size_sync = 0;
446 my $atime_sync = 0;
447 my $mtime_sync = 0;
448 my @sync_files;
449 my @ln_files;
450
451 foreach ($lc->get_intersection) {
452
453 my $f;
454
455 if ($file_size{$_} != $smb_size{$_}) {
456 $f=$_;
457 $size_sync++;
458 }
459 if ($file_atime{$_} != $smb_atime{$_}) {
460 $f=$_;
461 $atime_sync++;
462 }
463 if ($file_mtime{$_} != $smb_mtime{$_}) {
464 $f=$_;
465 $mtime_sync++;
466 }
467
468 if ($f) {
469 push @sync_files, $f;
470 } else {
471 push @ln_files, $_;
472 }
473 }
474
475 xlog($share,($#sync_files+1)." files will be updated (diff: $size_sync size, $atime_sync atime, $mtime_sync mtime), ".($#ln_files+1)." will be linked.");
476
477 foreach (@sync_files) {
478 $transfer += smb_copy($smb,"smb:$share",$bc,$_);
479 }
480
481 xlog($share,"$transfer bytes transfered...");
482
483 foreach (@ln_files) {
484 link "$real_bl/$_","$bc/$_" || warn "link $real_bl/$_ -> $bc/$_: $!\n";
485 }
486
487 # remove files
488 foreach (sort @files2erase) {
489 unlink "$bc/$_" || warn "unlink $_: $!\n";
490 }
491
492 # remove not needed dirs (after files)
493 foreach (sort @dirs2erase) {
494 rmdir "$bc/$_" || warn "rmdir $_: $!\n";
495 }
496
497 # remove old .md5sum
498 foreach (sort @dirs) {
499 unlink "$bc/$_/.md5sum" if (-e "$bc/$_/.md5sum");
500 }
501
502 # create .md5sum
503 my $last_dir = '';
504 my $md5;
505 foreach my $f (sort { $file_md5{$a} cmp $file_md5{$b} } keys %file_md5) {
506 my $dir = dirname($f);
507 my $file = basename($f);
508 #print "$f -- $dir / $file<--\n";
509 if ($dir ne $last_dir) {
510 close($md5) if ($md5);
511 open($md5, ">> $bc/$dir/.md5sum") || warn "can't create $bc/$dir/.md5sum: $!";
512 $last_dir = $dir;
513 #print STDERR "writing $last_dir/.md5sum\n";
514 }
515 print $md5 $file_md5{$f}," $file\n";
516 }
517 close($md5) if ($md5);
518
519 # create leatest link
520 #print "ln -s $bc $real_bl\n";
521 if (-l $bl) {
522 unlink $bl || warn "can't remove old latest symlink $bl: $!\n";
523 }
524 symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";
525
526 # FIX: sanity check -- remove for speedup
527 xlog($share,"failed to create latest symlink $bl -> $bc...") if (readlink($bl) ne $bc || ! -l $bl);
528
529 xlog($share,"backup completed...");
530 }
531
532 __END__
533 #-------------------------------------------------------------------------
534
535
536 =head1 NAME
537
538 psinib - Perl Snapshot Is Not Incremental Backup
539
540 =head1 SYNOPSIS
541
542 ./psinib.pl
543
544 =head1 DESCRIPTION
545
546 This script in current version support just backup of Samba (or Micro$oft
547 Winblowz) shares to central disk space. Central disk space is organized in
548 multiple directories named after:
549
550 =over 4
551
552 =item *
553 server which is sharing files to be backed up
554
555 =item *
556 name of share on server
557
558 =item *
559 dated directory named like standard ISO date format (YYYYMMDD).
560
561 =back
562
563 In each dated directory you will find I<snapshot> of all files on
564 exported share on that particular date.
565
566 You can also use symlink I<latest> which will lead you to
567 last completed backup. After that you can use some other backup
568 software to transfer I<snapshot> to tape, CD-ROM or some other media.
569
570 =head2 Design considerations
571
572 Since taking of share snapshot every day requires a lot of disk space and
573 network bandwidth, B<psinib> uses several techniques to keep disk usage and
574 network traffic at acceptable level:
575
576 =over 3
577
578 =item - usage of hard-links to provide same files in each snapshot (as opposed
579 to have multiple copies of same file)
580
581 =item - usage of file size, atime and mtime to find changes of files without
582 transferring whole file over network (just share browsing is transfered
583 over network)
584
585 =item - usage of C<.md5sum> files (compatible with command-line utility
586 C<md5sum>) to keep file between snapshots hard-linked
587
588 =back
589
590 =head1 CONFIGURATION
591
592 This section is not yet written.
593
594 =head1 HACKS, TRICKS, BUGS and LIMITATIONS
595
596 This chapter will have all content that doesn't fit anywhere else.
597
598 =head2 Can snapshots be more frequent than daily?
599
600 There is not real reason why you can't take snapshot more often than
601 once a day. Actually, if you are using B<psinib> to backup Windows
602 workstations you already know that they tend to come-and-go during the day
603 (reboots probably ;-), so running B<psinib> several times a day increases
604 your chance of having up-to-date backup (B<psinib> will not make multiple
605 snapshots for same day, nor will it update snapshot for current day if
606 it already exists).
607
608 However, changing B<psinib> to produce snapshots which are, for example, hourly
609 is a simple change of C<$DIR_TIME_FMT> which is currently set to
610 C<'%Y%m%d'> (see I<strftime> documentation for explanation of that
611 format). If you change that to C<'%Y%m%d-%H> you can have hourly snapshots
612 (if your network is fast enough, that is...). Also, some of messages in
613 program will sound strange, but other than that it should work.
614 I<You have been warned>.
615
616 =head2 Do I really need to share every directory which I want to snapshot?
617
618 Actually, no. Due to usage of C<Filesys::SmbClient> module, you can also
619 specify sub-directory inside your share that you want to backup. This feature
620 is most useful if you want to use administrative shares (but, have in mind
621 that you have to enter your Win administrator password in unencrypted file on
622 disk to do that) like this:
623
624 smbmount //server/c$/WinNT/fonts /mnt -o username=administrator%win
625
626 After that you will get directories with snapshots like:
627
628 server/c_WinNT_fonts/yyyymmdd/....
629
630 =head2 Won't I run out of disk space?
631
632 Of course you will... Snapshots and logfiles will eventually fill-up your disk.
633 However, you can do two things to stop that:
634
635 =head3 Clean snapshort older than x days
636
637 You can add following command to your C<root> crontab:
638
639 find /backup/isis_backup -type d -mindepth 3 -maxdepth 3 -mtime +11 -exec rm -Rf {} \;
640
641 I assume that C</backup/isis_backup> is directory in which are your snapshots
642 and that you don't want to keep snapshots older than 11 days (that's
643 C<-mtime +11> part of command).
644
645 =head3 Rotate your logs
646
647 I will leave that to you. I relay on GNU/Debian's C<logrotate> to do it for me.
648
649 =head2 What are I<YYYYMMDD.partial> directories?
650
651 If there isn't I<latest> symlink in snapshot directory, it's preatty safe to
652 assume that previous backup from that day failed. So, that directory will
653 be renamed to I<YYYYMMDD.partial> and snapshot will be performed again,
654 linking same files (other alternative would be to erase that dir and find
655 second-oldest directory, but this seemed like more correct approach).
656
657 =head1 AUTHOR
658
659 Dobrica Pavlinusic <dpavlin@rot13.org>
660
661 L<http://www.rot13.org/~dpavlin/>
662
663 =head1 LICENSE
664
665 This product is licensed under GNU Public License (GPL) v2 or later.
666
667 =cut

  ViewVC Help
Powered by ViewVC 1.1.26