15 |
use Cwd qw/abs_path/; |
use Cwd qw/abs_path/; |
16 |
|
|
17 |
use constant BPC_FTYPE_DIR => 5; |
use constant BPC_FTYPE_DIR => 5; |
18 |
use constant EST_CHUNK => 100000; |
use constant EST_CHUNK => 4096; |
19 |
|
|
20 |
# daylight saving time change offset for 1h |
# daylight saving time change offset for 1h |
21 |
my $dst_offset = 60 * 60; |
my $dst_offset = 60 * 60; |
58 |
|
|
59 |
my %opt; |
my %opt; |
60 |
|
|
61 |
if ( !getopts("cdm:v:ijf", \%opt ) ) { |
if ( !getopts("cdm:v:ijfq", \%opt ) ) { |
62 |
print STDERR <<EOF; |
print STDERR <<EOF; |
63 |
usage: $0 [-c|-d] [-m num] [-v|-v level] [-i|-j|-f] |
usage: $0 [-c|-d] [-m num] [-v|-v level] [-i|-j|-f] |
64 |
|
|
70 |
-i update Hyper Estraier full text index |
-i update Hyper Estraier full text index |
71 |
-j update full text, don't check existing files |
-j update full text, don't check existing files |
72 |
-f don't do anything with full text index |
-f don't do anything with full text index |
73 |
|
-q be quiet for hosts without changes |
74 |
|
|
75 |
Option -j is variation on -i. It will allow faster initial creation |
Option -j is variation on -i. It will allow faster initial creation |
76 |
of full-text index from existing database. |
of full-text index from existing database. |
105 |
return strftime($t_fmt,localtime()); |
return strftime($t_fmt,localtime()); |
106 |
} |
} |
107 |
|
|
|
my $hest_db; |
|
108 |
my $hest_node; |
my $hest_node; |
109 |
|
|
|
sub signal { |
|
|
my($sig) = @_; |
|
|
if ($hest_db) { |
|
|
print "\nCaught a SIG$sig--syncing database and shutting down\n"; |
|
|
$hest_db->sync(); |
|
|
$hest_db->close(); |
|
|
} |
|
|
exit(0); |
|
|
} |
|
|
|
|
|
$SIG{'INT'} = \&signal; |
|
|
$SIG{'QUIT'} = \&signal; |
|
|
|
|
110 |
sub hest_update { |
sub hest_update { |
111 |
|
|
112 |
my ($host_id, $share_id, $num) = @_; |
my ($host_id, $share_id, $num) = @_; |
113 |
|
|
114 |
my $skip_check = $opt{j} && print STDERR "Skipping check for existing files -- this should be used only with initital import\n"; |
my $skip_check = $opt{j} && print STDERR "Skipping check for existing files -- this should be used only with initital import\n"; |
115 |
|
|
116 |
unless (defined($index_node_url)) { |
unless ($index_node_url && $index_node_url =~ m#^http://#) { |
117 |
print STDERR "HyperEstraier support not enabled in configuration\n"; |
print STDERR "HyperEstraier support not enabled or index node invalid\n" if ($debug); |
118 |
$index_node_url = 0; |
$index_node_url = 0; |
119 |
return; |
return; |
120 |
} |
} |
126 |
my $offset = 0; |
my $offset = 0; |
127 |
my $added = 0; |
my $added = 0; |
128 |
|
|
|
print " opening index $index_node_url"; |
|
129 |
if ($index_node_url) { |
if ($index_node_url) { |
130 |
$hest_node ||= Search::Estraier::Node->new($index_node_url); |
print " opening index $index_node_url"; |
131 |
$hest_node->set_auth('admin', 'admin'); |
$hest_node ||= Search::Estraier::Node->new( |
132 |
|
url => $index_node_url, |
133 |
|
user => 'admin', |
134 |
|
passwd => 'admin', |
135 |
|
croak_on_error => 1, |
136 |
|
); |
137 |
print " via node URL"; |
print " via node URL"; |
|
} else { |
|
|
die "don't know how to use Hyper Estraier Index $index_node_url"; |
|
138 |
} |
} |
139 |
|
|
140 |
my $results = 0; |
my $results = 0; |
182 |
|
|
183 |
if ($results == 0) { |
if ($results == 0) { |
184 |
print " - no new files\n"; |
print " - no new files\n"; |
185 |
last; |
return; |
186 |
} else { |
} else { |
187 |
print " - $results files: "; |
print "..."; |
188 |
} |
} |
189 |
|
|
190 |
sub fmt_date { |
sub fmt_date { |
196 |
|
|
197 |
while (my $row = $sth->fetchrow_hashref()) { |
while (my $row = $sth->fetchrow_hashref()) { |
198 |
|
|
199 |
my $fid = $row->{'fid'} || die "no fid?"; |
my $uri = $row->{hname} . ':' . $row->{sname} . '#' . $row->{backupnum} . ' ' . $row->{filepath}; |
200 |
my $uri = 'file:///' . $fid; |
if (! $skip_check && $hest_node) { |
201 |
|
my $id = $hest_node->uri_to_id($uri); |
202 |
unless ($skip_check) { |
next if ($id && $id == -1); |
|
my $id = ($hest_db || $hest_node)->uri_to_id($uri); |
|
|
next unless ($id == -1); |
|
203 |
} |
} |
204 |
|
|
205 |
# create a document object |
# create a document object |
224 |
print STDERR $doc->dump_draft,"\n" if ($debug > 1); |
print STDERR $doc->dump_draft,"\n" if ($debug > 1); |
225 |
|
|
226 |
# register the document object to the database |
# register the document object to the database |
227 |
if ($hest_node) { |
$hest_node->put_doc($doc) if ($hest_node); |
228 |
$hest_node->put_doc($doc); |
|
|
} else { |
|
|
die "not supported"; |
|
|
} |
|
229 |
$added++; |
$added++; |
230 |
} |
} |
231 |
|
|
232 |
print " $added"; |
print "$added"; |
233 |
|
|
234 |
$offset += EST_CHUNK; |
$offset += EST_CHUNK; |
235 |
|
|
299 |
size bigint not null, |
size bigint not null, |
300 |
inc_size bigint not null default -1, |
inc_size bigint not null default -1, |
301 |
inc_deleted boolean default false, |
inc_deleted boolean default false, |
302 |
parts integer not null default 1, |
parts integer not null default 0, |
303 |
PRIMARY KEY(id) |
PRIMARY KEY(id) |
304 |
); |
); |
305 |
|
|
379 |
$dbh->do( qq{ CREATE SEQUENCE $seq } ); |
$dbh->do( qq{ CREATE SEQUENCE $seq } ); |
380 |
} |
} |
381 |
|
|
382 |
|
print " creating triggers "; |
383 |
|
$dbh->do( <<__END_OF_TRIGGER__ ); |
384 |
|
|
385 |
|
create or replace function backup_parts_check() returns trigger as ' |
386 |
|
declare |
387 |
|
b_parts integer; |
388 |
|
b_counted integer; |
389 |
|
b_id integer; |
390 |
|
begin |
391 |
|
-- raise notice ''old/new parts %/% backup_id %/%'', old.parts, new.parts, old.id, new.id; |
392 |
|
if (TG_OP=''UPDATE'') then |
393 |
|
b_id := new.id; |
394 |
|
b_parts := new.parts; |
395 |
|
elsif (TG_OP = ''INSERT'') then |
396 |
|
b_id := new.id; |
397 |
|
b_parts := new.parts; |
398 |
|
end if; |
399 |
|
b_counted := (select count(*) from backup_parts where backup_id = b_id); |
400 |
|
-- raise notice ''backup % parts %'', b_id, b_parts; |
401 |
|
if ( b_parts != b_counted ) then |
402 |
|
raise exception ''Update of backup % aborted, requested % parts and there are really % parts'', b_id, b_parts, b_counted; |
403 |
|
end if; |
404 |
|
return null; |
405 |
|
end; |
406 |
|
' language plpgsql; |
407 |
|
|
408 |
|
create trigger do_backup_parts_check |
409 |
|
after insert or update or delete on backups |
410 |
|
for each row execute procedure backup_parts_check(); |
411 |
|
|
412 |
|
create or replace function backup_backup_parts_check() returns trigger as ' |
413 |
|
declare |
414 |
|
b_id integer; |
415 |
|
my_part_nr integer; |
416 |
|
calc_part integer; |
417 |
|
begin |
418 |
|
if (TG_OP = ''INSERT'') then |
419 |
|
-- raise notice ''trigger: % backup_id %'', TG_OP, new.backup_id; |
420 |
|
b_id = new.backup_id; |
421 |
|
my_part_nr = new.part_nr; |
422 |
|
execute ''update backups set parts = parts + 1 where id = '' || b_id; |
423 |
|
elsif (TG_OP = ''DELETE'') then |
424 |
|
-- raise notice ''trigger: % backup_id %'', TG_OP, old.backup_id; |
425 |
|
b_id = old.backup_id; |
426 |
|
my_part_nr = old.part_nr; |
427 |
|
execute ''update backups set parts = parts - 1 where id = '' || b_id; |
428 |
|
end if; |
429 |
|
calc_part := (select count(part_nr) from backup_parts where backup_id = b_id); |
430 |
|
if ( my_part_nr != calc_part ) then |
431 |
|
raise exception ''Update of backup_parts with backup_id % aborted, requested part_nr is % and calulated next is %'', b_id, my_part_nr, calc_part; |
432 |
|
end if; |
433 |
|
return null; |
434 |
|
end; |
435 |
|
' language plpgsql; |
436 |
|
|
437 |
|
create trigger do_backup_backup_parts_check |
438 |
|
after insert or update or delete on backup_parts |
439 |
|
for each row execute procedure backup_backup_parts_check(); |
440 |
|
|
441 |
|
__END_OF_TRIGGER__ |
442 |
|
|
443 |
print "...\n"; |
print "...\n"; |
444 |
|
|
516 |
} |
} |
517 |
|
|
518 |
$host_nr++; |
$host_nr++; |
|
print "host ", $hosts->{$host_key}->{'host'}, " [", |
|
|
$host_nr, "/", ($#hosts + 1), "]: "; |
|
|
|
|
519 |
# get backups for a host |
# get backups for a host |
520 |
my @backups = $bpc->BackupInfoRead($hostname); |
my @backups = $bpc->BackupInfoRead($hostname); |
521 |
my $incs = scalar @backups; |
my $incs = scalar @backups; |
|
print "$incs increments\n"; |
|
522 |
|
|
523 |
|
my $host_header = sprintf("host %s [%d/%d]: %d increments\n", |
524 |
|
$hosts->{$host_key}->{'host'}, |
525 |
|
$host_nr, |
526 |
|
($#hosts + 1), |
527 |
|
$incs |
528 |
|
); |
529 |
|
print $host_header unless ($opt{q}); |
530 |
|
|
531 |
my $inc_nr = 0; |
my $inc_nr = 0; |
532 |
$beenThere = {}; |
$beenThere = {}; |
533 |
|
|
539 |
my $backupNum = $backup->{'num'}; |
my $backupNum = $backup->{'num'}; |
540 |
my @backupShares = (); |
my @backupShares = (); |
541 |
|
|
542 |
printf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n", |
my $share_header = sprintf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n", |
543 |
$hosts->{$host_key}->{'host'}, |
$hosts->{$host_key}->{'host'}, |
544 |
$inc_nr, $incs, $backupNum, |
$inc_nr, $incs, $backupNum, |
545 |
$backup->{type} || '?', |
$backup->{type} || '?', |
547 |
strftime($t_fmt,localtime($backup->{startTime})), |
strftime($t_fmt,localtime($backup->{startTime})), |
548 |
fmt_time($backup->{endTime} - $backup->{startTime}) |
fmt_time($backup->{endTime} - $backup->{startTime}) |
549 |
); |
); |
550 |
|
print $share_header unless ($opt{q}); |
551 |
|
|
552 |
my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1); |
my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1); |
553 |
foreach my $share ($files->shareList($backupNum)) { |
foreach my $share ($files->shareList($backupNum)) { |
561 |
# skip if allready in database! |
# skip if allready in database! |
562 |
next if ($count > 0); |
next if ($count > 0); |
563 |
|
|
564 |
|
# dump host and share header for -q |
565 |
|
if ($opt{q}) { |
566 |
|
if ($host_header) { |
567 |
|
print $host_header; |
568 |
|
$host_header = undef; |
569 |
|
} |
570 |
|
print $share_header; |
571 |
|
} |
572 |
|
|
573 |
# dump some log |
# dump some log |
574 |
print curr_time," ", $share; |
print curr_time," ", $share; |
575 |
|
|