--- trunk/bin/BackupPC_updatedb	2005/09/11 13:05:06	117
+++ trunk/bin/BackupPC_updatedb	2005/12/09 12:49:55	246
@@ -11,10 +11,14 @@
 use Time::HiRes qw/time/;
 use File::Pid;
 use POSIX qw/strftime/;
+use BackupPC::SearchLib;
 
 use constant BPC_FTYPE_DIR => 5;
 use constant EST_CHUNK => 100000;
 
+# daylight saving time change offset for 1h
+my $dst_offset = 60 * 60;
+
 my $debug = 0;
 $|=1;
 
@@ -43,22 +47,31 @@
 my $user = $Conf{SearchUser} || '';
 
 my $use_hest = $Conf{HyperEstraierIndex};
-my ($index_path, $index_node_url) = getHyperEstraier_url($use_hest);
+my ($index_path, $index_node_url) = BackupPC::SearchLib::getHyperEstraier_url($use_hest);
 
 my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
 
 my %opt;
 
-if ( !getopts("cdm:v:i", \%opt ) ) {
+if ( !getopts("cdm:v:ijf", \%opt ) ) {
 	print STDERR <<EOF;
-usage: $0 [-c|-d] [-m num] [-v|-v level] [-i]
+usage: $0 [-c|-d] [-m num] [-v|-v level] [-i|-j|-f]
 
 Options:
 	-c	create database on first use
 	-d	delete database before import
 	-m num	import just num	increments for one host
 	-v num	set verbosity (debug) level (default $debug)
-	-i	update HyperEstraier full text index
+	-i	update Hyper Estraier full text index
+	-j	update full text, don't check existing files
+	-f	don't do anything with full text index
+
+Option -j is variation on -i. It will allow faster initial creation
+of full-text index from existing database.
+
+Option -f will create database which is out of sync with full text index. You
+will have to re-run $0 with -i to fix it.
+
 EOF
 	exit 1;
 }
@@ -66,6 +79,9 @@
 if ($opt{v}) {
 	print "Debug level at $opt{v}\n";
 	$debug = $opt{v};
+} elsif ($opt{f}) {
+	print "WARNING: disabling full-text index update. You need to re-run $0 -j !\n";
+	($use_hest, $index_path, $index_node_url) = (undef, undef, undef);
 }
 
 #---- subs ----
@@ -103,6 +119,8 @@
 
 	my ($host_id, $share_id, $num) = @_;
 
+	my $skip_check = $opt{j} && print STDERR "Skipping check for existing files -- this should be used only with initital import\n";
+
 	unless ($use_hest) {
 		print STDERR "HyperEstraier support not enabled in configuration\n";
 		return;
@@ -118,7 +136,7 @@
 	print " opening index $use_hest";
 	if ($index_path) {
 		$hest_db = HyperEstraier::Database->new();
-		$hest_db->open($index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
+		$hest_db->open($TopDir . $index_path, $HyperEstraier::Database::DBWRITER | $HyperEstraier::Database::DBCREAT);
 		print " directly";
 	} elsif ($index_node_url) {
 		$hest_node ||= HyperEstraier::Node->new($index_node_url);
@@ -135,7 +153,7 @@
 
 		my $where = '';
 		my @data;
-		if ($host_id && $share_id && $num) {
+		if (defined($host_id) && defined($share_id) && defined($num)) {
 			$where = qq{
 			WHERE
 				hosts.id = ? AND
@@ -189,8 +207,10 @@
 			my $fid = $row->{'fid'} || die "no fid?";
 			my $uri = 'file:///' . $fid;
 
-			my $id = ($hest_db || $hest_node)->uri_to_id($uri);
-			next unless ($id == -1);
+			unless ($skip_check) {
+				my $id = ($hest_db || $hest_node)->uri_to_id($uri);
+				next unless ($id == -1);
+			}
 
 			# create a document object 
 			my $doc = HyperEstraier::Document->new;
@@ -199,7 +219,7 @@
 			$doc->add_attr('@uri', $uri);
 
 			foreach my $c (@{ $sth->{NAME} }) {
-				$doc->add_attr($c, $row->{$c}) if ($row->{$c});
+				$doc->add_attr($c, $row->{$c}) if (defined($row->{$c}));
 			}
 
 			#$doc->add_attr('@cdate', fmt_date($row->{'date'}));
@@ -246,11 +266,12 @@
 
 
 ## update index ##
-if (($opt{i} || ($index_path && ! -e $index_path)) && !$opt{c}) {
+if (($opt{i} || $opt{j} || ($index_path && ! -e $TopDir . $index_path)) && !$opt{c}) {
 	# update all
 	print "force update of HyperEstraier index ";
-	print "importing existing data" unless (-e $index_path);
+	print "importing existing data" unless (-e $TopDir . $index_path);
 	print "by -i flag" if ($opt{i});
+	print "by -j flag" if ($opt{j});
 	print "\n";
 	hest_update();
 }
@@ -259,86 +280,127 @@
 if ($opt{c}) {
 	sub do_index {
 		my $index = shift || return;
-		my ($table,$col,$unique) = split(/_/, $index);
+		my ($table,$col,$unique) = split(/:/, $index);
 		$unique ||= '';
-		$index =~ s/,/_/g;
+		$index =~ s/\W+/_/g;
+		print "$index on $table($col)" . ( $unique ? "u" : "" ) . " ";
 		$dbh->do(qq{ create $unique index $index on $table($col) });
 	}
 
 	print "creating tables...\n";
-      
-	$dbh->do(qq{
+
+	$dbh->do( qq{
 		create table hosts (
 			ID	SERIAL		PRIMARY KEY,
 			name	VARCHAR(30)	NOT NULL,
 			IP	VARCHAR(15)
 		);	      
-	});
-	      
-	$dbh->do(qq{
+
 		create table shares (
 			ID	SERIAL		PRIMARY KEY,
 			hostID	INTEGER		NOT NULL references hosts(id),
 			name	VARCHAR(30)	NOT NULL,
-			share	VARCHAR(200)	NOT NULL,
-			localpath VARCHAR(200)      
+			share	VARCHAR(200)	NOT NULL
 		);	      
-	});
-	
-	$dbh->do(qq{
+
+		create table dvds (
+			ID	SERIAL		PRIMARY KEY, 
+			num     INTEGER		NOT NULL,
+			name	VARCHAR(255)	NOT NULL,
+			mjesto	VARCHAR(255)
+		);
+
 		create table backups (
+			id	serial,
 			hostID	INTEGER		NOT NULL references hosts(id),
 			num     INTEGER		NOT NULL,
 			date	integer		NOT NULL, 
 			type	CHAR(4) 	not null,
 			shareID	integer		not null references shares(id),
-			size	integer		not null,
-			PRIMARY KEY(hostID, num, shareID) 
+			size	bigint		not null,
+			inc_size bigint		not null default -1,
+			inc_deleted boolean	default false,
+			parts	integer		not null default 1,
+			PRIMARY KEY(id)
 		); 	      
-	});
 
-	#do_index('backups_hostid,num_unique');
+		create table files (
+			ID		SERIAL,
+			shareID		INTEGER	NOT NULL references shares(id),
+			backupNum	INTEGER	NOT NULL,
+			name		VARCHAR(255) NOT NULL,
+			path		VARCHAR(255) NOT NULL,
+			date		integer	NOT NULL,
+			type		INTEGER	NOT NULL,
+			size		bigint	NOT NULL,
+			primary key(id)
+		);
 
-	$dbh->do(qq{
-		create table dvds (
-			ID	SERIAL		PRIMARY KEY, 
-			num     INTEGER		NOT NULL,
-			name	VARCHAR(255)	NOT NULL,
-			mjesto	VARCHAR(255)
+		create table archive (
+			id 		serial,
+			dvd_nr		int not null,
+			total_size	bigint default -1,
+			note		text,
+			username	varchar(20) not null,
+			date		timestamp default now(),
+			primary key(id)
+		);	
+
+		create table archive_backup (
+			archive_id	int not null references archive(id) on delete cascade,
+			backup_id	int not null references backups(id),
+			primary key(archive_id, backup_id)
 		);
-	});
 
-	$dbh->do(qq{     
-		create table files (
-			ID	SERIAL		PRIMARY KEY,  
-			shareID	INTEGER		NOT NULL references shares(id),
-			backupNum  INTEGER	NOT NULL,
-			name       VARCHAR(255)	NOT NULL,
-			path       VARCHAR(255)	NOT NULL,
-			date       integer	NOT NULL,
-			type       INTEGER	NOT NULL,
-			size       INTEGER	NOT NULL,
-			dvdid      INTEGER	references dvds(id)	
+		create table archive_burned (
+			archive_id	int references archive(id),
+			date		timestamp default now(),
+			part		int not null default 1,
+			copy		int not null default 1,
+			iso_size bigint default -1
+		);
+
+		create table backup_parts (
+			id serial,
+			backup_id int references backups(id),
+			part_nr int not null check (part_nr > 0),
+			tar_size bigint not null check (tar_size > 0),
+			size bigint not null check (size > 0),
+			md5 text not null,
+			items int not null check (items > 0),
+			date timestamp default now(),
+			primary key(id)
 		);
 	});
 
-	print "creating indexes:";
+	print "creating indexes: ";
 
 	foreach my $index (qw(
-		hosts_name
-		backups_hostID
-		backups_num
-		shares_hostID
-		shares_name
-		files_shareID
-		files_path
-		files_name
-		files_date
-		files_size
+		hosts:name
+		backups:hostID
+		backups:num
+		backups:shareID
+		shares:hostID
+		shares:name
+		files:shareID
+		files:path
+		files:name
+		files:date
+		files:size
+		archive:dvd_nr
+		archive_burned:archive_id
+		backup_parts:backup_id,part_nr
 	)) {
-		print " $index";
 		do_index($index);
 	}
+
+	print " creating sequence: ";
+	foreach my $seq (qw/dvd_nr/) {
+		print "$seq ";
+		$dbh->do( qq{ CREATE SEQUENCE $seq } );
+	}
+
+
 	print "...\n";
 
 	$dbh->commit;
@@ -382,7 +444,12 @@
 
 $sth->{insert_backups} = $dbh->prepare(qq{
 INSERT INTO backups (hostID, num, date, type, shareid, size)
-VALUES (?,?,?,?,?,?)
+VALUES (?,?,?,?,?,-1)
+});
+
+$sth->{update_backups_size} = $dbh->prepare(qq{
+UPDATE backups SET size = ?
+WHERE hostID = ? and num = ? and date = ? and type =? and shareid = ?
 });
 
 $sth->{insert_files} = $dbh->prepare(qq{
@@ -448,15 +515,23 @@
 			# dump some log
 			print curr_time," ", $share;
 
-			my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
-
 			$sth->{insert_backups}->execute(
 				$hostID,
 				$backupNum,
 				$backup->{'endTime'},
-				$backup->{'type'},
+				substr($backup->{'type'},0,4),
 				$shareID,
+			);
+
+			my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
+
+			$sth->{update_backups_size}->execute(
 				$size,
+				$hostID,
+				$backupNum,
+				$backup->{'endTime'},
+				substr($backup->{'type'},0,4),
+				$shareID,
 			);
 
 			print " commit";
@@ -499,14 +574,14 @@
 
 	$sth->{insert_share} ||= $dbh->prepare(qq{
 		INSERT INTO shares 
-			(hostID,name,share,localpath) 
-		VALUES (?,?,?,?)
+			(hostID,name,share) 
+		VALUES (?,?,?)
 	});
 
 	my $drop_down = $hostname . '/' . $share;
 	$drop_down =~ s#//+#/#g;
 
-	$sth->{insert_share}->execute($hostID,$share, $drop_down ,undef);
+	$sth->{insert_share}->execute($hostID,$share, $drop_down);
 	return $dbh->last_insert_id(undef,undef,'shares',undef);
 }
 
@@ -523,12 +598,12 @@
 		SELECT 1 FROM files
 		WHERE shareID = ? and
 			path = ? and 
-			date = ? and
-			size = ?
+			size = ? and
+			( date = ? or date = ? or date = ? )
 		LIMIT 1
 	});
 
-	my @param = ($shareID,$path,$date,$size);
+	my @param = ($shareID,$path,$size,$date, $date-$dst_offset, $date+$dst_offset);
 	$sth->{file_in_db}->execute(@param);
 	my $rows = $sth->{file_in_db}->rows;
 	print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
@@ -578,8 +653,29 @@
 				$filesInBackup->{$path_key}->{'size'}
 			));
 
+			my $key_dst_prev = join(" ", (
+				$shareID,
+				$dir,
+				$path_key,
+				$filesInBackup->{$path_key}->{'mtime'} - $dst_offset,
+				$filesInBackup->{$path_key}->{'size'}
+			));
+
+			my $key_dst_next = join(" ", (
+				$shareID,
+				$dir,
+				$path_key,
+				$filesInBackup->{$path_key}->{'mtime'} + $dst_offset,
+				$filesInBackup->{$path_key}->{'size'}
+			));
+
 			my $found;
-			if (! defined($beenThere->{$key}) && ! ($found = found_in_db($key, @data)) ) {
+			if (
+				! defined($beenThere->{$key}) &&
+				! defined($beenThere->{$key_dst_prev}) &&
+				! defined($beenThere->{$key_dst_next}) &&
+				! ($found = found_in_db($key, @data))
+			) {
 				print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
 
 				if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {