--- trunk/bin/BackupPC_updatedb	2005/08/30 09:55:55	97
+++ trunk/bin/BackupPC_updatedb	2005/09/11 12:39:24	116
@@ -13,7 +13,7 @@
 use POSIX qw/strftime/;
 
 use constant BPC_FTYPE_DIR => 5;
-use constant EST_SYNC_EVERY => 10000;
+use constant EST_CHUNK => 100000;
 
 my $debug = 0;
 $|=1;
@@ -41,9 +41,19 @@
 
 my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
 my $user = $Conf{SearchUser} || '';
-my $index_path = $Conf{HyperEstraierIndex};
-$index_path = $TopDir . '/' . $index_path;
-$index_path =~ s#//#/#g;
+
+my $use_hest = $Conf{HyperEstraierIndex};
+my ($index_path, $index_node_url);
+if ($use_hest) {
+	use HyperEstraier;
+	if ($use_hest =~ m#^http://#) {
+		$index_node_url = $use_hest;
+	} else {
+		$index_path = $TopDir . '/' . $index_path;
+		$index_path =~ s#//#/#g;
+	}
+}
+print "-- $use_hest : $index_path OR $index_node_url --\n";
 
 
 my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
@@ -85,6 +95,7 @@
 }
 
 my $hest_db;
+my $hest_node;
 
 sub signal {
 	my($sig) = @_;
@@ -103,129 +114,140 @@
 
 	my ($host_id, $share_id, $num) = @_;
 
-	print curr_time," updating HyperEstraier: select files";
-
-	my $t = time();
-
-	my $where = '';
-	if ($host_id && $share_id && $num) {
-		$where = qq{
-		WHERE
-			hosts.id = ? AND
-			shares.id = ? AND
-			files.backupnum = ?
-		};
-	}
-
-	my $sth = $dbh->prepare(qq{
-		SELECT
-			files.id			AS fid,
-			hosts.name			AS hname,
-			shares.name			AS sname,
-			-- shares.share			AS sharename,
-			files.backupnum			AS backupnum,
-			-- files.name			AS filename,
-			files.path			AS filepath,
-			files.date			AS date,
-			files.type			AS type,
-			files.size			AS size,
-			files.shareid			AS shareid,
-			backups.date			AS backup_date
-		FROM files 
-			INNER JOIN shares	ON files.shareID=shares.ID
-			INNER JOIN hosts	ON hosts.ID = shares.hostID
-			INNER JOIN backups	ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID
-		$where
-	});
-
-	$sth->execute(@_);
-	my $results = $sth->rows;
-
-	if ($results == 0) {
-		print " - no files, skipping\n";
+	unless ($use_hest) {
+		print STDERR "HyperEstraier support not enabled in configuration\n";
 		return;
 	}
 
-	my $dot = int($results / 15) || 1;
-
-	print " $results ($dot/#)";
+	print curr_time," updating HyperEstraier:";
 
-	sub fmt_date {
-		my $t = shift || return;
-		my $iso = BackupPC::Lib::timeStamp($t);
-		$iso =~ s/\s/T/;
-		return $iso;
-	}
+	my $t = time();
 
-	my $max = int($results / $dot);
+	my $offset = 0;
+	my $added = 0;
 
-	print ", opening index $index_path...";
-	use HyperEstraier;
-	my $db = HyperEstraier::Database->new();
+	print " opening index $use_hest";
+	if ($index_path) {
+		$hest_db = HyperEstraier::Database->new();
+		$hest_db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
+		print " directly";
+	} elsif ($index_node_url) {
+		$hest_node ||= HyperEstraier::Node->new($index_node_url);
+		$hest_node->set_auth('admin', 'admin');
+		print " via node URL";
+	} else {
+		die "don't know how to use HyperEstraier Index $use_hest";
+	}
+	print " increment is " . EST_CHUNK . " files:";
 
-#	unless ($hest_db) {
-#		print " open reader";
-#		$hest_db = HyperEstraier::Database->new();
-#
-#	}
+	my $results = 0;
 
+	do {
 
-	$db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
+		my $where = '';
+		my @data;
+		if ($host_id && $share_id && $num) {
+			$where = qq{
+			WHERE
+				hosts.id = ? AND
+				shares.id = ? AND
+				files.backupnum = ?
+			};
+			@data = ( $host_id, $share_id, $num );
+		}
 
-	my $added = 0;
+		my $limit = sprintf('LIMIT '.EST_CHUNK.' OFFSET %d', $offset);
 
-	while (my $row = $sth->fetchrow_hashref()) {
+		my $sth = $dbh->prepare(qq{
+			SELECT
+				files.id			AS fid,
+				hosts.name			AS hname,
+				shares.name			AS sname,
+				-- shares.share			AS sharename,
+				files.backupnum			AS backupnum,
+				-- files.name			AS filename,
+				files.path			AS filepath,
+				files.date			AS date,
+				files.type			AS type,
+				files.size			AS size,
+				files.shareid			AS shareid,
+				backups.date			AS backup_date
+			FROM files 
+				INNER JOIN shares	ON files.shareID=shares.ID
+				INNER JOIN hosts	ON hosts.ID = shares.hostID
+				INNER JOIN backups	ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID
+			$where
+			$limit
+		});
+
+		$sth->execute(@data);
+		$results = $sth->rows;
+
+		if ($results == 0) {
+			print " - no new files\n";
+			last;
+		}
 
-		my $fid = $row->{'fid'} || die "no fid?";
-		my $uri = 'file:///' . $fid;
+		sub fmt_date {
+			my $t = shift || return;
+			my $iso = BackupPC::Lib::timeStamp($t);
+			$iso =~ s/\s/T/;
+			return $iso;
+		}
 
-		my $id = $db->uri_to_id($uri);
-		next unless ($id == -1);
+		while (my $row = $sth->fetchrow_hashref()) {
 
-		# create a document object 
-		my $doc = HyperEstraier::Document->new;
+			my $fid = $row->{'fid'} || die "no fid?";
+			my $uri = 'file:///' . $fid;
 
-		# add attributes to the document object 
-		$doc->add_attr('@uri', $uri);
+			my $id = ($hest_db || $hest_node)->uri_to_id($uri);
+			next unless ($id == -1);
 
-		foreach my $c (@{ $sth->{NAME} }) {
-			$doc->add_attr($c, $row->{$c}) if ($row->{$c});
+			# create a document object 
+			my $doc = HyperEstraier::Document->new;
+
+			# add attributes to the document object 
+			$doc->add_attr('@uri', $uri);
+
+			foreach my $c (@{ $sth->{NAME} }) {
+				$doc->add_attr($c, $row->{$c}) if ($row->{$c});
+			}
+
+			#$doc->add_attr('@cdate', fmt_date($row->{'date'}));
+
+			# add the body text to the document object 
+			my $path = $row->{'filepath'};
+			$doc->add_text($path);
+			$path =~ s/(.)/$1 /g;
+			$doc->add_hidden_text($path);
+
+			print STDERR $doc->dump_draft,"\n" if ($debug > 1);
+
+			# register the document object to the database
+			if ($hest_db) {
+				$hest_db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
+			} elsif ($hest_node) {
+				$hest_node->put_doc($doc);
+			} else {
+				die "not supported";
+			}
+			$added++;
 		}
 
-		#$doc->add_attr('@cdate', fmt_date($row->{'date'}));
+		print " $added";
+		$hest_db->sync() if ($index_path);
 
-		# add the body text to the document object 
-		my $path = $row->{'filepath'};
-		$doc->add_text($path);
-		$path =~ s/(.)/$1 /g;
-		$doc->add_hidden_text($path);
-
-		print STDERR $doc->dump_draft,"\n" if ($debug > 1);
-
-		# register the document object to the database
-		$db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
-
-		$added++;
-		if ($added % $dot == 0) {
-			print "$max ";
-			$max--;
-		}
+		$offset += EST_CHUNK;
 
-		if ($added % EST_SYNC_EVERY == 0) {
-			print "sync ";
-			$db->sync();
-		}
+	} while ($results == EST_CHUNK);
 
+	if ($index_path) {
+		print ", close";
+		$hest_db->close();
 	}
 
-	print "sync $added new files";
-	$db->sync();
-	print ", close";
-	$db->close();
-
 	my $dur = (time() - $t) || 1;
-	printf(" [%.2f/s new %.2f/s dur: %s]\n",
-		( $results / $dur ),
+	printf(" [%.2f/s dur: %s]\n",
 		( $added / $dur ),
 		fmt_time($dur)
 	);
@@ -459,7 +481,7 @@
 				fmt_time($dur)
 			);
 
-			hest_update($hostID, $shareID, $backupNum);
+			hest_update($hostID, $shareID, $backupNum) if ($nf + $nd > 0);
 		}
 
 	}