--- trunk/bin/BackupPC_updatedb	2005/08/21 23:25:47	65
+++ trunk/bin/BackupPC_updatedb	2005/08/28 08:40:06	81
@@ -45,19 +45,112 @@
 
 my %opt;
 
-if ( !getopts("cdm:v:", \%opt ) ) {
+if ( !getopts("cdm:v:i", \%opt ) ) {
 	print STDERR <<EOF;
-usage: $0 [-c|-d] [-m num] [-v|-v level]
+usage: $0 [-c|-d] [-m num] [-v|-v level] [-i]
 
 Options:
 	-c	create database on first use
 	-d	delete database before import
 	-m num	import just num	increments for one host
 	-v num	set verbosity (debug) level (default $debug)
+	-i	update HyperEstraier full text index
 EOF
 	exit 1;
 }
 
+if ($opt{v}) {
+	print "Debug level at $opt{v}\n";
+	$debug = $opt{v};
+}
+
+## update index ##
+if ($opt{i}) {
+
+	my $index_dir = '/var/tmp/casket';
+
+	print "updating HyperEstraier index $index_dir...";
+
+	use HyperEstraier;
+	my $db = HyperEstraier::Database->new();
+	$db->open($index_dir, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
+
+	my $sth = $dbh->prepare(qq{
+		SELECT
+			files.id			AS fid,
+			hosts.name			AS hname,
+			shares.name			AS sname,
+			shares.share			AS sharename,
+			files.backupNum			AS backupNum,
+			files.name			AS filename,
+			files.path			AS filepath,
+			files.date			AS date,
+			files.type			AS filetype,
+			files.size			AS size,
+			files.shareid			AS shareid
+		FROM files 
+			INNER JOIN shares	ON files.shareID=shares.ID
+			INNER JOIN hosts	ON hosts.ID = shares.hostID
+			INNER JOIN backups	ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID
+	});
+
+	$sth->execute();
+
+	my $dot = int($sth->rows / 15);
+
+	print $sth->rows, " files ($dot/#) ";
+
+	sub fmt_date {
+		my $t = shift || return;
+		my $iso = BackupPC::Lib::timeStamp($t);
+		$iso =~ s/\s/T/;
+		return $iso;
+	}
+
+	my $i = 0;
+	my $max = int($sth->rows / $dot);
+
+	while (my $row = $sth->fetchrow_hashref()) {
+
+		# create a document object 
+		my $doc = HyperEstraier::Document->new;
+
+		# add attributes to the document object 
+		$doc->add_attr('@uri', 'file:///' . $row->{'fid'});
+
+		foreach my $c (qw/fid hname sname sharename backupNum filename filepath shareid/) {
+			$doc->add_attr($c, $row->{$c}) if ($row->{$c});
+		}
+
+		$doc->add_attr('date', fmt_date($row->{'date'}));
+
+		# add the body text to the document object 
+		my $path = $row->{'filepath'};
+		$doc->add_text($path);
+		$path =~ s/(.)/$1 /g;
+		$doc->add_hidden_text($path);
+
+		print STDERR $doc->dump_draft,"\n" if ($debug > 1);
+
+		# register the document object to the database
+		$db->put_doc($doc, $HyperEstraier::Database::PDCLEAN);
+
+		$i++;
+		if ($i % $dot == 0) {
+			print "$max ";
+			$max--;
+		}
+
+	}
+
+	print "sync";
+	$db->sync();
+	print " close\n";
+	$db->close();
+
+	exit;
+}
+
 ###################################create tables############################3
 
 if ($opt{c}) {
@@ -96,6 +189,7 @@
 			date	integer		NOT NULL, 
 			type	CHAR(4) 	not null,
 			shareID	integer		not null references shares(id),
+			size	integer		not null,
 			PRIMARY KEY(hostID, num, shareID) 
 		); 	      
 	});
@@ -159,11 +253,6 @@
 	$dbh->commit;
 }
 
-if ($opt{v}) {
-	print "Debug level at $opt{v}\n";
-	$debug = $opt{v};
-}
-
 #################################INSERT VALUES#############################
 
 # get hosts
@@ -188,8 +277,8 @@
 });
 
 $sth->{insert_backups} = $dbh->prepare(qq{
-INSERT INTO backups (hostID, num, date, type, shareid)
-VALUES (?,?,?,?,?)
+INSERT INTO backups (hostID, num, date, type, shareid, size)
+VALUES (?,?,?,?,?,?)
 });
 
 $sth->{insert_files} = $dbh->prepare(qq{
@@ -222,13 +311,15 @@
 		$hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
 	}
 
-	print("host ".$hosts->{$host_key}->{'host'}.": ");
+	print "host ".$hosts->{$host_key}->{'host'}.": ";
  
 	# get backups for a host
 	my @backups = $bpc->BackupInfoRead($hostname);
-	print scalar @backups, " increments\n";
+	my $incs = scalar @backups;
+	print  "$incs increments\n";
 
 	my $inc_nr = 0;
+	$beenThere = {};
 
 	foreach my $backup (@backups) {
 
@@ -238,14 +329,14 @@
 		my $backupNum = $backup->{'num'};
 		my @backupShares = ();
 
-		print $hosts->{$host_key}->{'host'},
-			"\t#$backupNum\t", $backup->{type} || '?', " ",
-			$backup->{nFilesNew} || '?', "/", $backup->{nFiles} || '?',
-			" files (date: ",
+		printf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n", 
+			$hosts->{$host_key}->{'host'},
+			$inc_nr, $incs, $backupNum, 
+			$backup->{type} || '?',
+			$backup->{nFilesNew} || '?', $backup->{nFiles} || '?',
 			strftime($t_fmt,localtime($backup->{startTime})),
-			" dur: ",
-			fmt_time($backup->{endTime} - $backup->{startTime}),
-			")\n";
+			fmt_time($backup->{endTime} - $backup->{startTime})
+		);
 
 		my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
 		foreach my $share ($files->shareList($backupNum)) {
@@ -262,22 +353,24 @@
 			# dump some log
 			print strftime($t_fmt,localtime())," ", $share;
 
-			my ($f, $nf, $d, $nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
+			my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
 
 			$sth->{insert_backups}->execute(
 				$hostID,
 				$backupNum,
 				$backup->{'endTime'},
 				$backup->{'type'},
-				$shareID
+				$shareID,
+				$size,
 			);
 
 			print " commit";
 			$dbh->commit();
 
 			my $dur = (time() - $t) || 1;
-			printf(" %d/%d files %d/%d dirs [%.2f/s dur: %s]\n",
+			printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n",
 				$nf, $f, $nd, $d,
+				($size / 1024 / 1024),
 				( ($f+$d) / $dur ),
 				fmt_time($dur)
 			);
@@ -325,7 +418,7 @@
 	my @data = @_;
 	shift @data;
 
-	my ($key, $shareID,undef,$name,$path,undef,$date,undef,$size) = @_;
+	my ($key, $shareID,undef,$name,$path,$date,undef,$size) = @_;
 
 	return $beenThere->{$key} if (defined($beenThere->{$key}));
 
@@ -333,15 +426,15 @@
 		SELECT 1 FROM files
 		WHERE shareID = ? and
 			path = ? and 
-			name = ? and
 			date = ? and
 			size = ?
+		LIMIT 1
 	});
 
-	my @param = ($shareID,$path,$name,$date,$size);
+	my @param = ($shareID,$path,$date,$size);
 	$sth->{file_in_db}->execute(@param);
 	my $rows = $sth->{file_in_db}->rows;
-	print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
+	print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
 
 	$beenThere->{$key}++;
 
@@ -359,7 +452,7 @@
 
 	print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);
 
-	my ($nr_files, $new_files, $nr_dirs, $new_dirs) = (0,0,0,0);
+	my ($nr_files, $new_files, $nr_dirs, $new_dirs, $size) = (0,0,0,0,0);
 
 	{ # scope
 		my @stack;
@@ -369,6 +462,7 @@
 
 		# first, add all the entries in current directory
 		foreach my $path_key (keys %{$filesInBackup}) {
+			print STDERR "# file ",Dumper($filesInBackup->{$path_key}),"\n" if ($debug >= 3);
 			my @data = (
 				$shareID,
 				$backupNum,
@@ -387,17 +481,18 @@
 				$filesInBackup->{$path_key}->{'size'}
 			));
 
-
-			if (! defined($beenThere->{$key}) && ! found_in_db($key, @data)) {
+			my $found;
+			if (! defined($beenThere->{$key}) && ! ($found = found_in_db($key, @data)) ) {
 				print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
 
 				if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
-					$new_dirs++;
+					$new_dirs++ unless ($found);
 					print STDERR " dir\n" if ($debug >= 2);
 				} else {
-					$new_files++;
+					$new_files++ unless ($found);
 					print STDERR " file\n" if ($debug >= 2);
 				}
+				$size += $filesInBackup->{$path_key}->{'size'} || 0;
 			}
 
 			if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
@@ -422,15 +517,16 @@
 		print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);
 
 		while ( my $dir = shift @stack ) {
-			my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
+			my ($f,$nf,$d,$nd, $s) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
 			print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
 			$nr_files += $f;
 			$new_files += $nf;
 			$nr_dirs += $d;
 			$new_dirs += $nd;
+			$size += $s;
 		}
 	}
 
-	return ($nr_files, $new_files, $nr_dirs, $new_dirs);
+	return ($nr_files, $new_files, $nr_dirs, $new_dirs, $size);
 }