--- trunk/bin/BackupPC_updatedb	2005/08/22 00:09:59	66
+++ trunk/bin/BackupPC_updatedb	2005/09/11 12:39:24	116
@@ -13,6 +13,7 @@
 use POSIX qw/strftime/;
 
 use constant BPC_FTYPE_DIR => 5;
+use constant EST_CHUNK => 100000;
 
 my $debug = 0;
 $|=1;
@@ -41,25 +42,231 @@
 my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
 my $user = $Conf{SearchUser} || '';
 
+my $use_hest = $Conf{HyperEstraierIndex};
+my ($index_path, $index_node_url);
+if ($use_hest) {
+	use HyperEstraier;
+	if ($use_hest =~ m#^http://#) {
+		$index_node_url = $use_hest;
+	} else {
+		$index_path = $TopDir . '/' . $index_path;
+		$index_path =~ s#//#/#g;
+	}
+}
+print "-- $use_hest : $index_path OR $index_node_url --\n";
+
+
 my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
 
 my %opt;
 
-if ( !getopts("cdm:v:", \%opt ) ) {
+if ( !getopts("cdm:v:i", \%opt ) ) {
 	print STDERR <<EOF;
-usage: $0 [-c|-d] [-m num] [-v|-v level]
+usage: $0 [-c|-d] [-m num] [-v|-v level] [-i]
 
 Options:
 	-c	create database on first use
 	-d	delete database before import
 	-m num	import just num	increments for one host
 	-v num	set verbosity (debug) level (default $debug)
+	-i	update HyperEstraier full text index
 EOF
 	exit 1;
 }
 
-###################################create tables############################3
+if ($opt{v}) {
+	print "Debug level at $opt{v}\n";
+	$debug = $opt{v};
+}
+
+#---- subs ----
+
+sub fmt_time {
+	my $t = shift || return;
+	my $out = "";
+	my ($ss,$mm,$hh) = gmtime($t);
+	$out .= "${hh}h" if ($hh);
+	$out .= sprintf("%02d:%02d", $mm,$ss);
+	return $out;
+}
+
+sub curr_time {
+	return strftime($t_fmt,localtime());
+}
+
+my $hest_db;
+my $hest_node;
+
+sub signal {
+	my($sig) = @_;
+	if ($hest_db) {
+		print "\nCaught a SIG$sig--syncing database and shutting down\n";
+		$hest_db->sync();
+		$hest_db->close();
+	}
+	exit(0);
+}
+
+$SIG{'INT'}  = \&signal;
+$SIG{'QUIT'} = \&signal;
+
+sub hest_update {
+
+	my ($host_id, $share_id, $num) = @_;
+
+	unless ($use_hest) {
+		print STDERR "HyperEstraier support not enabled in configuration\n";
+		return;
+	}
+
+	print curr_time," updating HyperEstraier:";
+
+	my $t = time();
+
+	my $offset = 0;
+	my $added = 0;
+
+	print " opening index $use_hest";
+	if ($index_path) {
+		$hest_db = HyperEstraier::Database->new();
+		$hest_db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
+		print " directly";
+	} elsif ($index_node_url) {
+		$hest_node ||= HyperEstraier::Node->new($index_node_url);
+		$hest_node->set_auth('admin', 'admin');
+		print " via node URL";
+	} else {
+		die "don't know how to use HyperEstraier Index $use_hest";
+	}
+	print " increment is " . EST_CHUNK . " files:";
+
+	my $results = 0;
+
+	do {
+
+		my $where = '';
+		my @data;
+		if ($host_id && $share_id && $num) {
+			$where = qq{
+			WHERE
+				hosts.id = ? AND
+				shares.id = ? AND
+				files.backupnum = ?
+			};
+			@data = ( $host_id, $share_id, $num );
+		}
+
+		my $limit = sprintf('LIMIT '.EST_CHUNK.' OFFSET %d', $offset);
 
+		my $sth = $dbh->prepare(qq{
+			SELECT
+				files.id			AS fid,
+				hosts.name			AS hname,
+				shares.name			AS sname,
+				-- shares.share			AS sharename,
+				files.backupnum			AS backupnum,
+				-- files.name			AS filename,
+				files.path			AS filepath,
+				files.date			AS date,
+				files.type			AS type,
+				files.size			AS size,
+				files.shareid			AS shareid,
+				backups.date			AS backup_date
+			FROM files 
+				INNER JOIN shares	ON files.shareID=shares.ID
+				INNER JOIN hosts	ON hosts.ID = shares.hostID
+				INNER JOIN backups	ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID
+			$where
+			$limit
+		});
+
+		$sth->execute(@data);
+		$results = $sth->rows;
+
+		if ($results == 0) {
+			print " - no new files\n";
+			last;
+		}
+
+		sub fmt_date {
+			my $t = shift || return;
+			my $iso = BackupPC::Lib::timeStamp($t);
+			$iso =~ s/\s/T/;
+			return $iso;
+		}
+
+		while (my $row = $sth->fetchrow_hashref()) {
+
+			my $fid = $row->{'fid'} || die "no fid?";
+			my $uri = 'file:///' . $fid;
+
+			my $id = ($hest_db || $hest_node)->uri_to_id($uri);
+			next unless ($id == -1);
+
+			# create a document object 
+			my $doc = HyperEstraier::Document->new;
+
+			# add attributes to the document object 
+			$doc->add_attr('@uri', $uri);
+
+			foreach my $c (@{ $sth->{NAME} }) {
+				$doc->add_attr($c, $row->{$c}) if ($row->{$c});
+			}
+
+			#$doc->add_attr('@cdate', fmt_date($row->{'date'}));
+
+			# add the body text to the document object 
+			my $path = $row->{'filepath'};
+			$doc->add_text($path);
+			$path =~ s/(.)/$1 /g;
+			$doc->add_hidden_text($path);
+
+			print STDERR $doc->dump_draft,"\n" if ($debug > 1);
+
+			# register the document object to the database
+			if ($hest_db) {
+				$hest_db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
+			} elsif ($hest_node) {
+				$hest_node->put_doc($doc);
+			} else {
+				die "not supported";
+			}
+			$added++;
+		}
+
+		print " $added";
+		$hest_db->sync() if ($index_path);
+
+		$offset += EST_CHUNK;
+
+	} while ($results == EST_CHUNK);
+
+	if ($index_path) {
+		print ", close";
+		$hest_db->close();
+	}
+
+	my $dur = (time() - $t) || 1;
+	printf(" [%.2f/s dur: %s]\n",
+		( $added / $dur ),
+		fmt_time($dur)
+	);
+}
+
+#---- /subs ----
+
+
+## update index ##
+if (($opt{i} || ($index_path && ! -e $index_path)) && !$opt{c}) {
+	# update all
+	print "force update of HyperEstraier index ";
+	print "importing existing data" unless (-e $index_path);
+	print "by -i flag" if ($opt{i});
+	print "\n";
+	hest_update();
+}
+
+## create tables ##
 if ($opt{c}) {
 	sub do_index {
 		my $index = shift || return;
@@ -149,6 +356,7 @@
 
 }
 
+## delete data before inseting ##
 if ($opt{d}) {
 	print "deleting ";
 	foreach my $table (qw(files dvds backups shares hosts)) {
@@ -160,12 +368,7 @@
 	$dbh->commit;
 }
 
-if ($opt{v}) {
-	print "Debug level at $opt{v}\n";
-	$debug = $opt{v};
-}
-
-#################################INSERT VALUES#############################
+## insert new values ##
 
 # get hosts
 $hosts = $bpc->HostInfoRead();
@@ -199,15 +402,6 @@
 	VALUES (?,?,?,?,?,?,?)
 });
 
-sub fmt_time {
-	my $t = shift || return;
-	my $out = "";
-	my ($ss,$mm,$hh) = gmtime($t);
-	$out .= "${hh}h" if ($hh);
-	$out .= sprintf("%02d:%02d", $mm,$ss);
-	return $out;
-}
-
 foreach my $host_key (keys %{$hosts}) {
 
 	my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key";
@@ -223,13 +417,15 @@
 		$hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
 	}
 
-	print("host ".$hosts->{$host_key}->{'host'}.": ");
+	print "host ".$hosts->{$host_key}->{'host'}.": ";
  
 	# get backups for a host
 	my @backups = $bpc->BackupInfoRead($hostname);
-	print scalar @backups, " increments\n";
+	my $incs = scalar @backups;
+	print  "$incs increments\n";
 
 	my $inc_nr = 0;
+	$beenThere = {};
 
 	foreach my $backup (@backups) {
 
@@ -239,14 +435,14 @@
 		my $backupNum = $backup->{'num'};
 		my @backupShares = ();
 
-		print $hosts->{$host_key}->{'host'},
-			"\t#$backupNum\t", $backup->{type} || '?', " ",
-			$backup->{nFilesNew} || '?', "/", $backup->{nFiles} || '?',
-			" files (date: ",
+		printf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n", 
+			$hosts->{$host_key}->{'host'},
+			$inc_nr, $incs, $backupNum, 
+			$backup->{type} || '?',
+			$backup->{nFilesNew} || '?', $backup->{nFiles} || '?',
 			strftime($t_fmt,localtime($backup->{startTime})),
-			" dur: ",
-			fmt_time($backup->{endTime} - $backup->{startTime}),
-			")\n";
+			fmt_time($backup->{endTime} - $backup->{startTime})
+		);
 
 		my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
 		foreach my $share ($files->shareList($backupNum)) {
@@ -261,7 +457,7 @@
 			next if ($count > 0);
 
 			# dump some log
-			print strftime($t_fmt,localtime())," ", $share;
+			print curr_time," ", $share;
 
 			my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
 
@@ -284,6 +480,8 @@
 				( ($f+$d) / $dur ),
 				fmt_time($dur)
 			);
+
+			hest_update($hostID, $shareID, $backupNum) if ($nf + $nd > 0);
 		}
 
 	}
@@ -328,7 +526,7 @@
 	my @data = @_;
 	shift @data;
 
-	my ($key, $shareID,undef,$name,$path,undef,$date,undef,$size) = @_;
+	my ($key, $shareID,undef,$name,$path,$date,undef,$size) = @_;
 
 	return $beenThere->{$key} if (defined($beenThere->{$key}));
 
@@ -336,15 +534,15 @@
 		SELECT 1 FROM files
 		WHERE shareID = ? and
 			path = ? and 
-			name = ? and
 			date = ? and
 			size = ?
+		LIMIT 1
 	});
 
-	my @param = ($shareID,$path,$name,$date,$size);
+	my @param = ($shareID,$path,$date,$size);
 	$sth->{file_in_db}->execute(@param);
 	my $rows = $sth->{file_in_db}->rows;
-	print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
+	print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
 
 	$beenThere->{$key}++;
 
@@ -391,15 +589,15 @@
 				$filesInBackup->{$path_key}->{'size'}
 			));
 
-
-			if (! defined($beenThere->{$key}) && ! found_in_db($key, @data)) {
+			my $found;
+			if (! defined($beenThere->{$key}) && ! ($found = found_in_db($key, @data)) ) {
 				print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
 
 				if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
-					$new_dirs++;
+					$new_dirs++ unless ($found);
 					print STDERR " dir\n" if ($debug >= 2);
 				} else {
-					$new_files++;
+					$new_files++ unless ($found);
 					print STDERR " file\n" if ($debug >= 2);
 				}
 				$size += $filesInBackup->{$path_key}->{'size'} || 0;