--- trunk/bin/BackupPC_updatedb	2005/08/28 09:12:54	82
+++ trunk/bin/BackupPC_updatedb	2005/09/23 08:54:11	131
@@ -11,8 +11,10 @@
 use Time::HiRes qw/time/;
 use File::Pid;
 use POSIX qw/strftime/;
+use BackupPC::SearchLib;
 
 use constant BPC_FTYPE_DIR => 5;
+use constant EST_CHUNK => 100000;
 
 my $debug = 0;
 $|=1;
@@ -40,13 +42,15 @@
 
 my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
 my $user = $Conf{SearchUser} || '';
-my $index_path = $Conf{HyperEstraierIndex};
+
+my $use_hest = $Conf{HyperEstraierIndex};
+my ($index_path, $index_node_url) = BackupPC::SearchLib::getHyperEstraier_url($use_hest);
 
 my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
 
 my %opt;
 
-if ( !getopts("cdm:v:i", \%opt ) ) {
+if ( !getopts("cdm:v:ij", \%opt ) ) {
 	print STDERR <<EOF;
 usage: $0 [-c|-d] [-m num] [-v|-v level] [-i]
 
@@ -55,7 +59,12 @@
 	-d	delete database before import
 	-m num	import just num	increments for one host
 	-v num	set verbosity (debug) level (default $debug)
-	-i	update HyperEstraier full text index
+	-i	update Hyper Estraier full text index
+	-j	update full text, don't check existing files
+
+Option -j is variation on -i. It will allow faster initial creation
+of full-text index from existing database.
+
 EOF
 	exit 1;
 }
@@ -65,98 +74,199 @@
 	$debug = $opt{v};
 }
 
-## update index ##
-if ($opt{i}) {
+#---- subs ----
 
-	print "updating HyperEstraier files ";
-	
-	my $sth = $dbh->prepare(qq{
-		SELECT
-			files.id			AS fid,
-			hosts.name			AS hname,
-			shares.name			AS sname,
-			shares.share			AS sharename,
-			files.backupNum			AS backupNum,
-			files.name			AS filename,
-			files.path			AS filepath,
-			files.date			AS date,
-			files.type			AS filetype,
-			files.size			AS size,
-			files.shareid			AS shareid
-		FROM files 
-			INNER JOIN shares	ON files.shareID=shares.ID
-			INNER JOIN hosts	ON hosts.ID = shares.hostID
-			INNER JOIN backups	ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID
-	});
-
-	$sth->execute();
-
-	my $dot = int($sth->rows / 15);
-
-	print $sth->rows, " files ($dot/#) ";
-
-	sub fmt_date {
-		my $t = shift || return;
-		my $iso = BackupPC::Lib::timeStamp($t);
-		$iso =~ s/\s/T/;
-		return $iso;
+sub fmt_time {
+	my $t = shift || return;
+	my $out = "";
+	my ($ss,$mm,$hh) = gmtime($t);
+	$out .= "${hh}h" if ($hh);
+	$out .= sprintf("%02d:%02d", $mm,$ss);
+	return $out;
+}
+
+sub curr_time {
+	return strftime($t_fmt,localtime());
+}
+
+my $hest_db;
+my $hest_node;
+
+sub signal {
+	my($sig) = @_;
+	if ($hest_db) {
+		print "\nCaught a SIG$sig--syncing database and shutting down\n";
+		$hest_db->sync();
+		$hest_db->close();
+	}
+	exit(0);
+}
+
+$SIG{'INT'}  = \&signal;
+$SIG{'QUIT'} = \&signal;
+
+sub hest_update {
+
+	my ($host_id, $share_id, $num) = @_;
+
+	my $skip_check = $opt{j} && print STDERR "Skipping check for existing files -- this should be used only with initital import\n";
+
+	unless ($use_hest) {
+		print STDERR "HyperEstraier support not enabled in configuration\n";
+		return;
 	}
 
-	my $i = 0;
-	my $max = int($sth->rows / $dot);
+	print curr_time," updating HyperEstraier:";
+
+	my $t = time();
 
-	$index_path = $TopDir . '/' . $index_path;
-	$index_path =~ s#//#/#g;
+	my $offset = 0;
+	my $added = 0;
 
-	print "index $index_path...";
-	use HyperEstraier;
-	my $db = HyperEstraier::Database->new();
-	$db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
+	print " opening index $use_hest";
+	if ($index_path) {
+		$hest_db = HyperEstraier::Database->new();
+		$hest_db->open($TopDir . $index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
+		print " directly";
+	} elsif ($index_node_url) {
+		$hest_node ||= HyperEstraier::Node->new($index_node_url);
+		$hest_node->set_auth('admin', 'admin');
+		print " via node URL";
+	} else {
+		die "don't know how to use HyperEstraier Index $use_hest";
+	}
+	print " increment is " . EST_CHUNK . " files:";
 
+	my $results = 0;
 
-	while (my $row = $sth->fetchrow_hashref()) {
+	do {
 
-		# create a document object 
-		my $doc = HyperEstraier::Document->new;
+		my $where = '';
+		my @data;
+		if ($host_id && $share_id && $num) {
+			$where = qq{
+			WHERE
+				hosts.id = ? AND
+				shares.id = ? AND
+				files.backupnum = ?
+			};
+			@data = ( $host_id, $share_id, $num );
+		}
 
-		# add attributes to the document object 
-		$doc->add_attr('@uri', 'file:///' . $row->{'fid'});
+		my $limit = sprintf('LIMIT '.EST_CHUNK.' OFFSET %d', $offset);
 
-		foreach my $c (qw/fid hname sname sharename backupNum filename filepath shareid/) {
-			$doc->add_attr($c, $row->{$c}) if ($row->{$c});
+		my $sth = $dbh->prepare(qq{
+			SELECT
+				files.id			AS fid,
+				hosts.name			AS hname,
+				shares.name			AS sname,
+				-- shares.share			AS sharename,
+				files.backupnum			AS backupnum,
+				-- files.name			AS filename,
+				files.path			AS filepath,
+				files.date			AS date,
+				files.type			AS type,
+				files.size			AS size,
+				files.shareid			AS shareid,
+				backups.date			AS backup_date
+			FROM files 
+				INNER JOIN shares	ON files.shareID=shares.ID
+				INNER JOIN hosts	ON hosts.ID = shares.hostID
+				INNER JOIN backups	ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID
+			$where
+			$limit
+		});
+
+		$sth->execute(@data);
+		$results = $sth->rows;
+
+		if ($results == 0) {
+			print " - no new files\n";
+			last;
 		}
 
-		$doc->add_attr('date', fmt_date($row->{'date'}));
+		sub fmt_date {
+			my $t = shift || return;
+			my $iso = BackupPC::Lib::timeStamp($t);
+			$iso =~ s/\s/T/;
+			return $iso;
+		}
 
-		# add the body text to the document object 
-		my $path = $row->{'filepath'};
-		$doc->add_text($path);
-		$path =~ s/(.)/$1 /g;
-		$doc->add_hidden_text($path);
-
-		print STDERR $doc->dump_draft,"\n" if ($debug > 1);
-
-		# register the document object to the database
-		$db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
-
-		$i++;
-		if ($i % $dot == 0) {
-			print "$max ";
-			$max--;
+		while (my $row = $sth->fetchrow_hashref()) {
+
+			my $fid = $row->{'fid'} || die "no fid?";
+			my $uri = 'file:///' . $fid;
+
+			unless ($skip_check) {
+				my $id = ($hest_db || $hest_node)->uri_to_id($uri);
+				next unless ($id == -1);
+			}
+
+			# create a document object 
+			my $doc = HyperEstraier::Document->new;
+
+			# add attributes to the document object 
+			$doc->add_attr('@uri', $uri);
+
+			foreach my $c (@{ $sth->{NAME} }) {
+				$doc->add_attr($c, $row->{$c}) if ($row->{$c});
+			}
+
+			#$doc->add_attr('@cdate', fmt_date($row->{'date'}));
+
+			# add the body text to the document object 
+			my $path = $row->{'filepath'};
+			$doc->add_text($path);
+			$path =~ s/(.)/$1 /g;
+			$doc->add_hidden_text($path);
+
+			print STDERR $doc->dump_draft,"\n" if ($debug > 1);
+
+			# register the document object to the database
+			if ($hest_db) {
+				$hest_db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
+			} elsif ($hest_node) {
+				$hest_node->put_doc($doc);
+			} else {
+				die "not supported";
+			}
+			$added++;
 		}
 
-	}
+		print " $added";
+		$hest_db->sync() if ($index_path);
+
+		$offset += EST_CHUNK;
+
+	} while ($results == EST_CHUNK);
 
-	print "sync";
-	$db->sync();
-	print " close\n";
-	$db->close();
+	if ($index_path) {
+		print ", close";
+		$hest_db->close();
+	}
 
-	exit;
+	my $dur = (time() - $t) || 1;
+	printf(" [%.2f/s dur: %s]\n",
+		( $added / $dur ),
+		fmt_time($dur)
+	);
 }
 
-###################################create tables############################3
+#---- /subs ----
+
+
+## update index ##
+if (($opt{i} || $opt{j} || ($index_path && ! -e $index_path)) && !$opt{c}) {
+	# update all
+	print "force update of HyperEstraier index ";
+	print "importing existing data" unless (-e $index_path);
+	print "by -i flag" if ($opt{i});
+	print "by -j flag" if ($opt{j});
+	print "\n";
+	hest_update();
+}
 
+## create tables ##
 if ($opt{c}) {
 	sub do_index {
 		my $index = shift || return;
@@ -185,6 +295,15 @@
 			localpath VARCHAR(200)      
 		);	      
 	});
+
+	$dbh->do(qq{
+		create table dvds (
+			ID	SERIAL		PRIMARY KEY, 
+			num     INTEGER		NOT NULL,
+			name	VARCHAR(255)	NOT NULL,
+			mjesto	VARCHAR(255)
+		);
+	});
 	
 	$dbh->do(qq{
 		create table backups (
@@ -200,14 +319,6 @@
 
 	#do_index('backups_hostid,num_unique');
 
-	$dbh->do(qq{
-		create table dvds (
-			ID	SERIAL		PRIMARY KEY, 
-			num     INTEGER		NOT NULL,
-			name	VARCHAR(255)	NOT NULL,
-			mjesto	VARCHAR(255)
-		);
-	});
 
 	$dbh->do(qq{     
 		create table files (
@@ -218,11 +329,104 @@
 			path       VARCHAR(255)	NOT NULL,
 			date       integer	NOT NULL,
 			type       INTEGER	NOT NULL,
-			size       INTEGER	NOT NULL,
-			dvdid      INTEGER	references dvds(id)	
+			size       INTEGER	NOT NULL
+		);
+	});
+
+
+	$dbh->do( qq{
+		create table archive
+		(
+			id 			int not null,
+			dvd_nr		int not null,
+			note		text,
+			username	varchar(20) not null,
+			date		timestamp,
+			primary key(id)
+		);	
+	}
+	);
+
+	$dbh->do( qq{
+		create table archive_backup
+		(
+			archive_id	int not null,
+			backup_id	int not null,
+			status		text,
+			primary key(archive_id, backup_id)
+		);
+	});
+
+	$dbh->do( qq{
+		create table workflows(
+			id 			int not null,
+			step_id		int not null,
+			start		timestamp,
+			stop		timestamp,
+			username	varchar(20),
+			archive_id	int not null,
+			running 	boolean default true,
+			primary key(id)
+		);
+	});
+
+	$dbh->do( qq{
+		create table workflow_step
+		(
+			step_id		int not null,
+			code		text,
+			next_step	int,
+			stop		boolean default false,
+			primary key(step_id)
 		);
 	});
 
+	$dbh->do( qq{
+			alter table workflow_step
+				add constraint fk_workflow_next_step
+				foreign key(next_step)
+				references workflow_step(step_id);
+	});
+
+	$dbh->do( qq{
+		alter table workflows
+			add constraint fk_workflows_step_id
+			foreign key(step_id)
+			references workflow_step(step_id);
+	});
+	
+	$dbh->do( qq{
+		alter table workflows
+			add constraint fk_workflows_archive_id
+			foreign key(archive_id)
+			references archive(id);	
+	});
+
+	$dbh->do( qq{
+		create table workflow_log
+		(
+			workflow_id		int not null,
+			step_id			int not null,
+			date			timestamp not null,
+			status			text,
+			primary key(workflow_id, step_id)
+		);
+	});
+
+	$dbh->do( qq{
+		alter table workflow_log
+			add constraint fk_workflow_log_workflow_id
+			foreign key (workflow_id)
+			references workflows(id);
+		});
+	
+	$dbh->do( qq{
+		alter table workflow_log
+			add constraint fk_workflow_log_step_id
+			foreign key (step_id)
+			references 	workflow_step(step_id);
+		});
+
 	print "creating indexes:";
 
 	foreach my $index (qw(
@@ -246,6 +450,7 @@
 
 }
 
+## delete data before inseting ##
 if ($opt{d}) {
 	print "deleting ";
 	foreach my $table (qw(files dvds backups shares hosts)) {
@@ -257,7 +462,7 @@
 	$dbh->commit;
 }
 
-#################################INSERT VALUES#############################
+## insert new values ##
 
 # get hosts
 $hosts = $bpc->HostInfoRead();
@@ -291,15 +496,6 @@
 	VALUES (?,?,?,?,?,?,?)
 });
 
-sub fmt_time {
-	my $t = shift || return;
-	my $out = "";
-	my ($ss,$mm,$hh) = gmtime($t);
-	$out .= "${hh}h" if ($hh);
-	$out .= sprintf("%02d:%02d", $mm,$ss);
-	return $out;
-}
-
 foreach my $host_key (keys %{$hosts}) {
 
 	my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key";
@@ -355,7 +551,7 @@
 			next if ($count > 0);
 
 			# dump some log
-			print strftime($t_fmt,localtime())," ", $share;
+			print curr_time," ", $share;
 
 			my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
 
@@ -363,7 +559,7 @@
 				$hostID,
 				$backupNum,
 				$backup->{'endTime'},
-				$backup->{'type'},
+				substr($backup->{'type'},0,4),
 				$shareID,
 				$size,
 			);
@@ -378,6 +574,8 @@
 				( ($f+$d) / $dur ),
 				fmt_time($dur)
 			);
+
+			hest_update($hostID, $shareID, $backupNum) if ($nf + $nd > 0);
 		}
 
 	}