/[BackupPC]/trunk/bin/BackupPC_updatedb
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /trunk/bin/BackupPC_updatedb

Parent Directory Parent Directory | Revision Log Revision Log


Revision 305 - (hide annotations)
Sat Jan 28 19:49:18 2006 UTC (18 years, 4 months ago) by dpavlin
File size: 17005 byte(s)
be chatty on debug level 3
1 dpavlin 14 #!/usr/local/bin/perl -w
2 dpavlin 6
3     use strict;
4 dpavlin 48 use lib "__INSTALLDIR__/lib";
5    
6 dpavlin 6 use DBI;
7     use BackupPC::Lib;
8     use BackupPC::View;
9     use Data::Dumper;
10     use Getopt::Std;
11 dpavlin 37 use Time::HiRes qw/time/;
12 dpavlin 38 use File::Pid;
13 dpavlin 37 use POSIX qw/strftime/;
14 dpavlin 118 use BackupPC::SearchLib;
15 dpavlin 250 use Cwd qw/abs_path/;
16 dpavlin 48
17 dpavlin 6 use constant BPC_FTYPE_DIR => 5;
18 dpavlin 99 use constant EST_CHUNK => 100000;
19 dpavlin 6
20 dpavlin 246 # daylight saving time change offset for 1h
21     my $dst_offset = 60 * 60;
22    
23 dpavlin 30 my $debug = 0;
24 dpavlin 14 $|=1;
25 dpavlin 6
26 dpavlin 51 my $start_t = time();
27    
28 dpavlin 250 my $pid_path = abs_path($0);
29     $pid_path =~ s/\W+/_/g;
30 dpavlin 38
31 dpavlin 250 my $pidfile = new File::Pid({
32     file => "/tmp/$pid_path",
33     });
34    
35 dpavlin 38 if (my $pid = $pidfile->running ) {
36     die "$0 already running: $pid\n";
37     } elsif ($pidfile->pid ne $$) {
38     $pidfile->remove;
39     $pidfile = new File::Pid;
40     }
41 dpavlin 250 print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";
42 dpavlin 39 $pidfile->write;
43 dpavlin 38
44 dpavlin 37 my $t_fmt = '%Y-%m-%d %H:%M:%S';
45    
46 dpavlin 6 my $hosts;
47     my $bpc = BackupPC::Lib->new || die;
48     my %Conf = $bpc->Conf();
49     my $TopDir = $bpc->TopDir();
50 dpavlin 14 my $beenThere = {};
51 dpavlin 6
52 dpavlin 51 my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
53     my $user = $Conf{SearchUser} || '';
54 dpavlin 116
55 dpavlin 303 my $index_node_url = $Conf{HyperEstraierIndex};
56 dpavlin 6
57 dpavlin 49 my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
58    
59 dpavlin 6 my %opt;
60    
61 dpavlin 246 if ( !getopts("cdm:v:ijf", \%opt ) ) {
62 dpavlin 6 print STDERR <<EOF;
63 dpavlin 246 usage: $0 [-c|-d] [-m num] [-v|-v level] [-i|-j|-f]
64 dpavlin 6
65     Options:
66 dpavlin 14 -c create database on first use
67     -d delete database before import
68     -m num import just num increments for one host
69 dpavlin 44 -v num set verbosity (debug) level (default $debug)
70 dpavlin 131 -i update Hyper Estraier full text index
71     -j update full text, don't check existing files
72 dpavlin 246 -f don't do anything with full text index
73 dpavlin 131
74     Option -j is variation on -i. It will allow faster initial creation
75     of full-text index from existing database.
76    
77 dpavlin 246 Option -f will create database which is out of sync with full text index. You
78     will have to re-run $0 with -i to fix it.
79    
80 dpavlin 6 EOF
81     exit 1;
82     }
83    
84 dpavlin 81 if ($opt{v}) {
85     print "Debug level at $opt{v}\n";
86     $debug = $opt{v};
87 dpavlin 246 } elsif ($opt{f}) {
88     print "WARNING: disabling full-text index update. You need to re-run $0 -j !\n";
89 dpavlin 303 $index_node_url = undef;
90 dpavlin 81 }
91    
92 dpavlin 86 #---- subs ----
93    
94     sub fmt_time {
95     my $t = shift || return;
96     my $out = "";
97     my ($ss,$mm,$hh) = gmtime($t);
98     $out .= "${hh}h" if ($hh);
99     $out .= sprintf("%02d:%02d", $mm,$ss);
100     return $out;
101     }
102    
103     sub curr_time {
104     return strftime($t_fmt,localtime());
105     }
106    
107 dpavlin 89 my $hest_db;
108 dpavlin 116 my $hest_node;
109 dpavlin 86
110 dpavlin 97 sub signal {
111     my($sig) = @_;
112     if ($hest_db) {
113     print "\nCaught a SIG$sig--syncing database and shutting down\n";
114     $hest_db->sync();
115     $hest_db->close();
116     }
117     exit(0);
118     }
119    
120     $SIG{'INT'} = \&signal;
121     $SIG{'QUIT'} = \&signal;
122    
123 dpavlin 89 sub hest_update {
124 dpavlin 81
125 dpavlin 89 my ($host_id, $share_id, $num) = @_;
126    
127 dpavlin 131 my $skip_check = $opt{j} && print STDERR "Skipping check for existing files -- this should be used only with initital import\n";
128    
129 dpavlin 303 unless (defined($index_node_url)) {
130 dpavlin 116 print STDERR "HyperEstraier support not enabled in configuration\n";
131 dpavlin 303 $index_node_url = 0;
132 dpavlin 116 return;
133     }
134    
135 dpavlin 303 print curr_time," updating Hyper Estraier:";
136 dpavlin 249
137 dpavlin 86 my $t = time();
138 dpavlin 89
139 dpavlin 98 my $offset = 0;
140     my $added = 0;
141 dpavlin 89
142 dpavlin 303 print " opening index $index_node_url";
143     if ($index_node_url) {
144     $hest_node ||= Search::Estraier::Node->new($index_node_url);
145 dpavlin 116 $hest_node->set_auth('admin', 'admin');
146 dpavlin 130 print " via node URL";
147 dpavlin 116 } else {
148 dpavlin 303 die "don't know how to use Hyper Estraier Index $index_node_url";
149 dpavlin 116 }
150 dpavlin 99
151 dpavlin 98 my $results = 0;
152 dpavlin 81
153 dpavlin 98 do {
154 dpavlin 81
155 dpavlin 98 my $where = '';
156 dpavlin 104 my @data;
157 dpavlin 190 if (defined($host_id) && defined($share_id) && defined($num)) {
158 dpavlin 98 $where = qq{
159     WHERE
160     hosts.id = ? AND
161     shares.id = ? AND
162     files.backupnum = ?
163     };
164 dpavlin 104 @data = ( $host_id, $share_id, $num );
165 dpavlin 98 }
166 dpavlin 89
167 dpavlin 98 my $limit = sprintf('LIMIT '.EST_CHUNK.' OFFSET %d', $offset);
168 dpavlin 81
169 dpavlin 98 my $sth = $dbh->prepare(qq{
170     SELECT
171     files.id AS fid,
172     hosts.name AS hname,
173     shares.name AS sname,
174     -- shares.share AS sharename,
175     files.backupnum AS backupnum,
176     -- files.name AS filename,
177     files.path AS filepath,
178     files.date AS date,
179     files.type AS type,
180     files.size AS size,
181     files.shareid AS shareid,
182     backups.date AS backup_date
183     FROM files
184     INNER JOIN shares ON files.shareID=shares.ID
185     INNER JOIN hosts ON hosts.ID = shares.hostID
186     INNER JOIN backups ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID
187     $where
188     $limit
189     });
190 dpavlin 81
191 dpavlin 104 $sth->execute(@data);
192 dpavlin 98 $results = $sth->rows;
193 dpavlin 81
194 dpavlin 98 if ($results == 0) {
195 dpavlin 107 print " - no new files\n";
196 dpavlin 98 last;
197 dpavlin 303 } else {
198     print " - $results files: ";
199 dpavlin 98 }
200 dpavlin 89
201 dpavlin 98 sub fmt_date {
202     my $t = shift || return;
203     my $iso = BackupPC::Lib::timeStamp($t);
204     $iso =~ s/\s/T/;
205     return $iso;
206     }
207 dpavlin 89
208 dpavlin 98 while (my $row = $sth->fetchrow_hashref()) {
209 dpavlin 89
210 dpavlin 98 my $fid = $row->{'fid'} || die "no fid?";
211     my $uri = 'file:///' . $fid;
212 dpavlin 82
213 dpavlin 131 unless ($skip_check) {
214     my $id = ($hest_db || $hest_node)->uri_to_id($uri);
215     next unless ($id == -1);
216     }
217 dpavlin 82
218 dpavlin 98 # create a document object
219 dpavlin 303 my $doc = Search::Estraier::Document->new;
220 dpavlin 81
221 dpavlin 98 # add attributes to the document object
222     $doc->add_attr('@uri', $uri);
223 dpavlin 89
224 dpavlin 98 foreach my $c (@{ $sth->{NAME} }) {
225 dpavlin 305 print STDERR "attr $c = $row->{$c}\n" if ($debug > 2);
226 dpavlin 241 $doc->add_attr($c, $row->{$c}) if (defined($row->{$c}));
227 dpavlin 98 }
228 dpavlin 89
229 dpavlin 98 #$doc->add_attr('@cdate', fmt_date($row->{'date'}));
230 dpavlin 81
231 dpavlin 98 # add the body text to the document object
232     my $path = $row->{'filepath'};
233     $doc->add_text($path);
234     $path =~ s/(.)/$1 /g;
235     $doc->add_hidden_text($path);
236 dpavlin 81
237 dpavlin 98 print STDERR $doc->dump_draft,"\n" if ($debug > 1);
238    
239     # register the document object to the database
240 dpavlin 303 if ($hest_node) {
241 dpavlin 116 $hest_node->put_doc($doc);
242     } else {
243     die "not supported";
244     }
245 dpavlin 98 $added++;
246 dpavlin 81 }
247    
248 dpavlin 98 print " $added";
249 dpavlin 81
250 dpavlin 98 $offset += EST_CHUNK;
251 dpavlin 81
252 dpavlin 98 } while ($results == EST_CHUNK);
253 dpavlin 81
254 dpavlin 86 my $dur = (time() - $t) || 1;
255 dpavlin 98 printf(" [%.2f/s dur: %s]\n",
256 dpavlin 89 ( $added / $dur ),
257 dpavlin 86 fmt_time($dur)
258     );
259 dpavlin 89 }
260 dpavlin 86
261 dpavlin 89 #---- /subs ----
262    
263    
264     ## update index ##
265 dpavlin 303 if ( ( $opt{i} || $opt{j} ) && !$opt{c} ) {
266 dpavlin 89 # update all
267 dpavlin 303 print "force update of Hyper Estraier index ";
268 dpavlin 89 print "by -i flag" if ($opt{i});
269 dpavlin 131 print "by -j flag" if ($opt{j});
270 dpavlin 89 print "\n";
271     hest_update();
272 dpavlin 81 }
273    
274 dpavlin 89 ## create tables ##
275 dpavlin 14 if ($opt{c}) {
276 dpavlin 49 sub do_index {
277     my $index = shift || return;
278 dpavlin 152 my ($table,$col,$unique) = split(/:/, $index);
279 dpavlin 49 $unique ||= '';
280 dpavlin 152 $index =~ s/\W+/_/g;
281     print "$index on $table($col)" . ( $unique ? "u" : "" ) . " ";
282 dpavlin 49 $dbh->do(qq{ create $unique index $index on $table($col) });
283     }
284    
285 dpavlin 14 print "creating tables...\n";
286 dpavlin 183
287     $dbh->do( qq{
288 dpavlin 14 create table hosts (
289 dpavlin 49 ID SERIAL PRIMARY KEY,
290 dpavlin 14 name VARCHAR(30) NOT NULL,
291     IP VARCHAR(15)
292 dpavlin 6 );
293 dpavlin 183
294 dpavlin 14 create table shares (
295 dpavlin 49 ID SERIAL PRIMARY KEY,
296 dpavlin 14 hostID INTEGER NOT NULL references hosts(id),
297     name VARCHAR(30) NOT NULL,
298 dpavlin 152 share VARCHAR(200) NOT NULL
299 dpavlin 6 );
300 dpavlin 130
301     create table dvds (
302     ID SERIAL PRIMARY KEY,
303     num INTEGER NOT NULL,
304     name VARCHAR(255) NOT NULL,
305     mjesto VARCHAR(255)
306     );
307 dpavlin 183
308 dpavlin 14 create table backups (
309 dpavlin 152 id serial,
310 dpavlin 14 hostID INTEGER NOT NULL references hosts(id),
311     num INTEGER NOT NULL,
312 dpavlin 49 date integer NOT NULL,
313     type CHAR(4) not null,
314 dpavlin 65 shareID integer not null references shares(id),
315 dpavlin 136 size bigint not null,
316 dpavlin 152 inc_size bigint not null default -1,
317     inc_deleted boolean default false,
318 dpavlin 196 parts integer not null default 1,
319 dpavlin 152 PRIMARY KEY(id)
320 dpavlin 6 );
321    
322 dpavlin 14 create table files (
323 dpavlin 152 ID SERIAL,
324     shareID INTEGER NOT NULL references shares(id),
325     backupNum INTEGER NOT NULL,
326     name VARCHAR(255) NOT NULL,
327     path VARCHAR(255) NOT NULL,
328     date integer NOT NULL,
329     type INTEGER NOT NULL,
330     size bigint NOT NULL,
331     primary key(id)
332 dpavlin 6 );
333    
334 dpavlin 152 create table archive (
335     id serial,
336 dpavlin 130 dvd_nr int not null,
337 dpavlin 152 total_size bigint default -1,
338 dpavlin 130 note text,
339     username varchar(20) not null,
340 dpavlin 152 date timestamp default now(),
341 dpavlin 130 primary key(id)
342     );
343    
344 dpavlin 183 create table archive_backup (
345 dpavlin 160 archive_id int not null references archive(id) on delete cascade,
346 dpavlin 152 backup_id int not null references backups(id),
347 dpavlin 130 primary key(archive_id, backup_id)
348     );
349 dpavlin 183
350     create table archive_burned (
351 dpavlin 199 archive_id int references archive(id),
352 dpavlin 202 date timestamp default now(),
353 dpavlin 199 part int not null default 1,
354 dpavlin 202 copy int not null default 1,
355 dpavlin 196 iso_size bigint default -1
356 dpavlin 183 );
357    
358 dpavlin 235 create table backup_parts (
359     id serial,
360     backup_id int references backups(id),
361     part_nr int not null check (part_nr > 0),
362     tar_size bigint not null check (tar_size > 0),
363     size bigint not null check (size > 0),
364     md5 text not null,
365     items int not null check (items > 0),
366     date timestamp default now(),
367     primary key(id)
368     );
369 dpavlin 130 });
370    
371 dpavlin 153 print "creating indexes: ";
372 dpavlin 6
373 dpavlin 14 foreach my $index (qw(
374 dpavlin 152 hosts:name
375     backups:hostID
376     backups:num
377     backups:shareID
378     shares:hostID
379     shares:name
380     files:shareID
381     files:path
382     files:name
383     files:date
384     files:size
385     archive:dvd_nr
386 dpavlin 183 archive_burned:archive_id
387 dpavlin 235 backup_parts:backup_id,part_nr
388 dpavlin 14 )) {
389 dpavlin 49 do_index($index);
390 dpavlin 14 }
391 dpavlin 153
392     print " creating sequence: ";
393     foreach my $seq (qw/dvd_nr/) {
394     print "$seq ";
395     $dbh->do( qq{ CREATE SEQUENCE $seq } );
396     }
397    
398    
399 dpavlin 49 print "...\n";
400 dpavlin 14
401 dpavlin 49 $dbh->commit;
402 dpavlin 14
403     }
404    
405 dpavlin 89 ## delete data before inseting ##
406 dpavlin 14 if ($opt{d}) {
407     print "deleting ";
408 dpavlin 49 foreach my $table (qw(files dvds backups shares hosts)) {
409 dpavlin 14 print "$table ";
410     $dbh->do(qq{ DELETE FROM $table });
411     }
412     print " done...\n";
413 dpavlin 49
414 dpavlin 51 $dbh->commit;
415 dpavlin 14 }
416    
417 dpavlin 89 ## insert new values ##
418 dpavlin 6
419     # get hosts
420 dpavlin 8 $hosts = $bpc->HostInfoRead();
421 dpavlin 6 my $hostID;
422     my $shareID;
423    
424 dpavlin 14 my $sth;
425 dpavlin 6
426 dpavlin 14 $sth->{insert_hosts} = $dbh->prepare(qq{
427     INSERT INTO hosts (name, IP) VALUES (?,?)
428     });
429 dpavlin 6
430 dpavlin 14 $sth->{hosts_by_name} = $dbh->prepare(qq{
431     SELECT ID FROM hosts WHERE name=?
432     });
433    
434 dpavlin 65 $sth->{backups_count} = $dbh->prepare(qq{
435 dpavlin 14 SELECT COUNT(*)
436     FROM backups
437 dpavlin 65 WHERE hostID=? AND num=? AND shareid=?
438 dpavlin 14 });
439    
440     $sth->{insert_backups} = $dbh->prepare(qq{
441 dpavlin 66 INSERT INTO backups (hostID, num, date, type, shareid, size)
442 dpavlin 242 VALUES (?,?,?,?,?,-1)
443 dpavlin 14 });
444    
445 dpavlin 242 $sth->{update_backups_size} = $dbh->prepare(qq{
446     UPDATE backups SET size = ?
447     WHERE hostID = ? and num = ? and date = ? and type =? and shareid = ?
448     });
449    
450 dpavlin 14 $sth->{insert_files} = $dbh->prepare(qq{
451     INSERT INTO files
452 dpavlin 62 (shareID, backupNum, name, path, date, type, size)
453     VALUES (?,?,?,?,?,?,?)
454 dpavlin 14 });
455    
456 dpavlin 248 my @hosts = keys %{$hosts};
457     my $host_nr = 0;
458 dpavlin 14
459 dpavlin 248 foreach my $host_key (@hosts) {
460    
461 dpavlin 14 my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key";
462    
463     $sth->{hosts_by_name}->execute($hosts->{$host_key}->{'host'});
464    
465     unless (($hostID) = $sth->{hosts_by_name}->fetchrow_array()) {
466     $sth->{insert_hosts}->execute(
467     $hosts->{$host_key}->{'host'},
468     $hosts->{$host_key}->{'ip'}
469     );
470    
471 dpavlin 49 $hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
472 dpavlin 14 }
473    
474 dpavlin 248 $host_nr++;
475     print "host ", $hosts->{$host_key}->{'host'}, " [",
476     $host_nr, "/", ($#hosts + 1), "]: ";
477 dpavlin 6
478 dpavlin 14 # get backups for a host
479     my @backups = $bpc->BackupInfoRead($hostname);
480 dpavlin 67 my $incs = scalar @backups;
481     print "$incs increments\n";
482 dpavlin 6
483 dpavlin 14 my $inc_nr = 0;
484 dpavlin 67 $beenThere = {};
485 dpavlin 14
486     foreach my $backup (@backups) {
487 dpavlin 40
488 dpavlin 14 $inc_nr++;
489     last if ($opt{m} && $inc_nr > $opt{m});
490    
491     my $backupNum = $backup->{'num'};
492     my @backupShares = ();
493    
494 dpavlin 67 printf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n",
495     $hosts->{$host_key}->{'host'},
496     $inc_nr, $incs, $backupNum,
497     $backup->{type} || '?',
498     $backup->{nFilesNew} || '?', $backup->{nFiles} || '?',
499 dpavlin 57 strftime($t_fmt,localtime($backup->{startTime})),
500 dpavlin 67 fmt_time($backup->{endTime} - $backup->{startTime})
501     );
502 dpavlin 14
503 dpavlin 34 my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
504 dpavlin 14 foreach my $share ($files->shareList($backupNum)) {
505    
506 dpavlin 37 my $t = time();
507    
508 dpavlin 14 $shareID = getShareID($share, $hostID, $hostname);
509    
510 dpavlin 65 $sth->{backups_count}->execute($hostID, $backupNum, $shareID);
511     my ($count) = $sth->{backups_count}->fetchrow_array();
512     # skip if allready in database!
513     next if ($count > 0);
514    
515     # dump some log
516 dpavlin 86 print curr_time," ", $share;
517 dpavlin 65
518 dpavlin 242 $sth->{insert_backups}->execute(
519     $hostID,
520     $backupNum,
521     $backup->{'endTime'},
522     substr($backup->{'type'},0,4),
523     $shareID,
524     );
525    
526 dpavlin 66 my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
527 dpavlin 65
528 dpavlin 250 eval {
529     $sth->{update_backups_size}->execute(
530     $size,
531     $hostID,
532     $backupNum,
533     $backup->{'endTime'},
534     substr($backup->{'type'},0,4),
535     $shareID,
536     );
537     print " commit";
538     $dbh->commit();
539     };
540     if ($@) {
541     print " rollback";
542     $dbh->rollback();
543     }
544 dpavlin 65
545 dpavlin 50 my $dur = (time() - $t) || 1;
546 dpavlin 66 printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n",
547 dpavlin 37 $nf, $f, $nd, $d,
548 dpavlin 66 ($size / 1024 / 1024),
549 dpavlin 50 ( ($f+$d) / $dur ),
550     fmt_time($dur)
551 dpavlin 37 );
552 dpavlin 89
553 dpavlin 107 hest_update($hostID, $shareID, $backupNum) if ($nf + $nd > 0);
554 dpavlin 14 }
555 dpavlin 29
556 dpavlin 6 }
557     }
558 dpavlin 14 undef $sth;
559 dpavlin 6 $dbh->commit();
560     $dbh->disconnect();
561    
562 dpavlin 51 print "total duration: ",fmt_time(time() - $start_t),"\n";
563    
564 dpavlin 38 $pidfile->remove;
565    
566 dpavlin 14 sub getShareID() {
567 dpavlin 6
568 dpavlin 14 my ($share, $hostID, $hostname) = @_;
569    
570     $sth->{share_id} ||= $dbh->prepare(qq{
571     SELECT ID FROM shares WHERE hostID=? AND name=?
572     });
573    
574     $sth->{share_id}->execute($hostID,$share);
575    
576     my ($id) = $sth->{share_id}->fetchrow_array();
577    
578     return $id if (defined($id));
579    
580     $sth->{insert_share} ||= $dbh->prepare(qq{
581     INSERT INTO shares
582 dpavlin 152 (hostID,name,share)
583     VALUES (?,?,?)
584 dpavlin 14 });
585    
586 dpavlin 25 my $drop_down = $hostname . '/' . $share;
587     $drop_down =~ s#//+#/#g;
588    
589 dpavlin 152 $sth->{insert_share}->execute($hostID,$share, $drop_down);
590 dpavlin 49 return $dbh->last_insert_id(undef,undef,'shares',undef);
591 dpavlin 6 }
592    
593 dpavlin 14 sub found_in_db {
594    
595 dpavlin 48 my @data = @_;
596     shift @data;
597 dpavlin 14
598 dpavlin 74 my ($key, $shareID,undef,$name,$path,$date,undef,$size) = @_;
599 dpavlin 48
600     return $beenThere->{$key} if (defined($beenThere->{$key}));
601    
602 dpavlin 14 $sth->{file_in_db} ||= $dbh->prepare(qq{
603 dpavlin 48 SELECT 1 FROM files
604 dpavlin 14 WHERE shareID = ? and
605     path = ? and
606 dpavlin 246 size = ? and
607     ( date = ? or date = ? or date = ? )
608 dpavlin 74 LIMIT 1
609 dpavlin 14 });
610    
611 dpavlin 246 my @param = ($shareID,$path,$size,$date, $date-$dst_offset, $date+$dst_offset);
612 dpavlin 14 $sth->{file_in_db}->execute(@param);
613 dpavlin 48 my $rows = $sth->{file_in_db}->rows;
614 dpavlin 74 print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
615 dpavlin 48
616     $beenThere->{$key}++;
617    
618     $sth->{'insert_files'}->execute(@data) unless ($rows);
619 dpavlin 14 return $rows;
620 dpavlin 6 }
621    
622     ####################################################
623     # recursing through filesystem structure and #
624     # and returning flattened files list #
625     ####################################################
626 dpavlin 14 sub recurseDir($$$$$$$$) {
627 dpavlin 6
628 dpavlin 35 my ($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID) = @_;
629 dpavlin 14
630 dpavlin 44 print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);
631 dpavlin 29
632 dpavlin 66 my ($nr_files, $new_files, $nr_dirs, $new_dirs, $size) = (0,0,0,0,0);
633 dpavlin 14
634 dpavlin 27 { # scope
635 dpavlin 29 my @stack;
636 dpavlin 14
637 dpavlin 45 print STDERR "# dirAttrib($backupNum, $share, $dir)\n" if ($debug >= 2);
638 dpavlin 27 my $filesInBackup = $files->dirAttrib($backupNum, $share, $dir);
639 dpavlin 14
640 dpavlin 27 # first, add all the entries in current directory
641     foreach my $path_key (keys %{$filesInBackup}) {
642 dpavlin 66 print STDERR "# file ",Dumper($filesInBackup->{$path_key}),"\n" if ($debug >= 3);
643 dpavlin 27 my @data = (
644     $shareID,
645     $backupNum,
646     $path_key,
647     $filesInBackup->{$path_key}->{'relPath'},
648     $filesInBackup->{$path_key}->{'mtime'},
649     $filesInBackup->{$path_key}->{'type'},
650     $filesInBackup->{$path_key}->{'size'}
651     );
652    
653     my $key = join(" ", (
654     $shareID,
655     $dir,
656     $path_key,
657     $filesInBackup->{$path_key}->{'mtime'},
658     $filesInBackup->{$path_key}->{'size'}
659     ));
660    
661 dpavlin 242 my $key_dst_prev = join(" ", (
662 dpavlin 240 $shareID,
663     $dir,
664     $path_key,
665     $filesInBackup->{$path_key}->{'mtime'} - $dst_offset,
666     $filesInBackup->{$path_key}->{'size'}
667     ));
668    
669 dpavlin 242 my $key_dst_next = join(" ", (
670 dpavlin 240 $shareID,
671     $dir,
672     $path_key,
673     $filesInBackup->{$path_key}->{'mtime'} + $dst_offset,
674     $filesInBackup->{$path_key}->{'size'}
675     ));
676    
677 dpavlin 70 my $found;
678 dpavlin 240 if (
679     ! defined($beenThere->{$key}) &&
680 dpavlin 242 ! defined($beenThere->{$key_dst_prev}) &&
681     ! defined($beenThere->{$key_dst_next}) &&
682 dpavlin 240 ! ($found = found_in_db($key, @data))
683     ) {
684 dpavlin 30 print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
685 dpavlin 48
686 dpavlin 27 if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
687 dpavlin 70 $new_dirs++ unless ($found);
688 dpavlin 30 print STDERR " dir\n" if ($debug >= 2);
689 dpavlin 27 } else {
690 dpavlin 70 $new_files++ unless ($found);
691 dpavlin 30 print STDERR " file\n" if ($debug >= 2);
692 dpavlin 27 }
693 dpavlin 66 $size += $filesInBackup->{$path_key}->{'size'} || 0;
694 dpavlin 27 }
695    
696 dpavlin 14 if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
697 dpavlin 27 $nr_dirs++;
698    
699 dpavlin 29 my $full_path = $dir . '/' . $path_key;
700     push @stack, $full_path;
701 dpavlin 30 print STDERR "### store to stack: $full_path\n" if ($debug >= 3);
702 dpavlin 29
703 dpavlin 27 # my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $backups, $backupNum, $share, $path_key, $shareID) unless ($beenThere->{$key});
704     #
705     # $nr_files += $f;
706     # $new_files += $nf;
707     # $nr_dirs += $d;
708     # $new_dirs += $nd;
709    
710 dpavlin 14 } else {
711 dpavlin 27 $nr_files++;
712 dpavlin 14 }
713     }
714    
715 dpavlin 30 print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);
716 dpavlin 14
717 dpavlin 29 while ( my $dir = shift @stack ) {
718 dpavlin 66 my ($f,$nf,$d,$nd, $s) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
719 dpavlin 30 print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
720 dpavlin 29 $nr_files += $f;
721     $new_files += $nf;
722     $nr_dirs += $d;
723     $new_dirs += $nd;
724 dpavlin 66 $size += $s;
725 dpavlin 29 }
726 dpavlin 14 }
727    
728 dpavlin 66 return ($nr_files, $new_files, $nr_dirs, $new_dirs, $size);
729 dpavlin 6 }
730    

Properties

Name Value
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26