--- trunk/bin/BackupPC_updatedb 2005/09/11 11:48:16 114 +++ trunk/bin/BackupPC_updatedb 2005/09/23 08:54:11 131 @@ -11,6 +11,7 @@ use Time::HiRes qw/time/; use File::Pid; use POSIX qw/strftime/; +use BackupPC::SearchLib; use constant BPC_FTYPE_DIR => 5; use constant EST_CHUNK => 100000; @@ -41,19 +42,15 @@ my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n"; my $user = $Conf{SearchUser} || ''; -my $index_path = $Conf{HyperEstraierIndex}; -if ($index_path) { - $index_path = $TopDir . '/' . $index_path; - $index_path =~ s#//#/#g; - use HyperEstraier; -} +my $use_hest = $Conf{HyperEstraierIndex}; +my ($index_path, $index_node_url) = BackupPC::SearchLib::getHyperEstraier_url($use_hest); my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 }); my %opt; -if ( !getopts("cdm:v:i", \%opt ) ) { +if ( !getopts("cdm:v:ij", \%opt ) ) { print STDERR <new(); - $hest_db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT); - + print " opening index $use_hest"; + if ($index_path) { + $hest_db = HyperEstraier::Database->new(); + $hest_db->open($TopDir . $index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT); + print " directly"; + } elsif ($index_node_url) { + $hest_node ||= HyperEstraier::Node->new($index_node_url); + $hest_node->set_auth('admin', 'admin'); + print " via node URL"; + } else { + die "don't know how to use HyperEstraier Index $use_hest"; + } print " increment is " . EST_CHUNK . " files:"; my $results = 0; @@ -179,8 +197,10 @@ my $fid = $row->{'fid'} || die "no fid?"; my $uri = 'file:///' . $fid; - my $id = $hest_db->uri_to_id($uri); - next unless ($id == -1); + unless ($skip_check) { + my $id = ($hest_db || $hest_node)->uri_to_id($uri); + next unless ($id == -1); + } # create a document object my $doc = HyperEstraier::Document->new; @@ -203,19 +223,27 @@ print STDERR $doc->dump_draft,"\n" if ($debug > 1); # register the document object to the database - $hest_db->put_doc($doc, $HyperEstraier::Database::PDCLEAN); + if ($hest_db) { + $hest_db->put_doc($doc, $HyperEstraier::Database::PDCLEAN); + } elsif ($hest_node) { + $hest_node->put_doc($doc); + } else { + die "not supported"; + } $added++; } print " $added"; - $hest_db->sync(); + $hest_db->sync() if ($index_path); $offset += EST_CHUNK; } while ($results == EST_CHUNK); - print ", close"; - $hest_db->close(); + if ($index_path) { + print ", close"; + $hest_db->close(); + } my $dur = (time() - $t) || 1; printf(" [%.2f/s dur: %s]\n", @@ -228,11 +256,12 @@ ## update index ## -if (($opt{i} || ($index_path && ! -e $index_path)) && !$opt{c}) { +if (($opt{i} || $opt{j} || ($index_path && ! -e $index_path)) && !$opt{c}) { # update all print "force update of HyperEstraier index "; print "importing existing data" unless (-e $index_path); print "by -i flag" if ($opt{i}); + print "by -j flag" if ($opt{j}); print "\n"; hest_update(); } @@ -266,6 +295,15 @@ localpath VARCHAR(200) ); }); + + $dbh->do(qq{ + create table dvds ( + ID SERIAL PRIMARY KEY, + num INTEGER NOT NULL, + name VARCHAR(255) NOT NULL, + mjesto VARCHAR(255) + ); + }); $dbh->do(qq{ create table backups ( @@ -281,14 +319,6 @@ #do_index('backups_hostid,num_unique'); - $dbh->do(qq{ - create table dvds ( - ID SERIAL PRIMARY KEY, - num INTEGER NOT NULL, - name VARCHAR(255) NOT NULL, - mjesto VARCHAR(255) - ); - }); $dbh->do(qq{ create table files ( @@ -299,11 +329,104 @@ path VARCHAR(255) NOT NULL, date integer NOT NULL, type INTEGER NOT NULL, - size INTEGER NOT NULL, - dvdid INTEGER references dvds(id) + size INTEGER NOT NULL + ); + }); + + + $dbh->do( qq{ + create table archive + ( + id int not null, + dvd_nr int not null, + note text, + username varchar(20) not null, + date timestamp, + primary key(id) + ); + } + ); + + $dbh->do( qq{ + create table archive_backup + ( + archive_id int not null, + backup_id int not null, + status text, + primary key(archive_id, backup_id) + ); + }); + + $dbh->do( qq{ + create table workflows( + id int not null, + step_id int not null, + start timestamp, + stop timestamp, + username varchar(20), + archive_id int not null, + running boolean default true, + primary key(id) + ); + }); + + $dbh->do( qq{ + create table workflow_step + ( + step_id int not null, + code text, + next_step int, + stop boolean default false, + primary key(step_id) + ); + }); + + $dbh->do( qq{ + alter table workflow_step + add constraint fk_workflow_next_step + foreign key(next_step) + references workflow_step(step_id); + }); + + $dbh->do( qq{ + alter table workflows + add constraint fk_workflows_step_id + foreign key(step_id) + references workflow_step(step_id); + }); + + $dbh->do( qq{ + alter table workflows + add constraint fk_workflows_archive_id + foreign key(archive_id) + references archive(id); + }); + + $dbh->do( qq{ + create table workflow_log + ( + workflow_id int not null, + step_id int not null, + date timestamp not null, + status text, + primary key(workflow_id, step_id) ); }); + $dbh->do( qq{ + alter table workflow_log + add constraint fk_workflow_log_workflow_id + foreign key (workflow_id) + references workflows(id); + }); + + $dbh->do( qq{ + alter table workflow_log + add constraint fk_workflow_log_step_id + foreign key (step_id) + references workflow_step(step_id); + }); + print "creating indexes:"; foreach my $index (qw( @@ -436,7 +559,7 @@ $hostID, $backupNum, $backup->{'endTime'}, - $backup->{'type'}, + substr($backup->{'type'},0,4), $shareID, $size, );