--- trunk/all2xml.pl	2004/12/31 03:35:43	618
+++ trunk/all2xml.pl	2005/01/23 15:18:03	643
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 
 use strict;
-use OpenIsis;
+use Biblio::Isis;
 use Getopt::Std;
 use Data::Dumper;
 use XML::Simple;
@@ -22,7 +22,8 @@
 my $config;
 
 #use index_DBI;		# default DBI module for index
-use index_DBI_cache;	# faster DBI module using memory cache
+#use index_DBI_cache;	# faster DBI module using memory cache
+use index_DBI_filter;	# filter support for indexes
 my $index;
 
 my %opts;
@@ -257,8 +258,6 @@
 			# init vars so that we go into while...
 			($swish,$display) = (1,1);
 
-			# placeholder for all repeatable entries for index
-
 			sub mkformat($$) {
 				my $x = shift || die "mkformat needs tag reference";
 				my $data = shift || return;
@@ -687,7 +686,10 @@
 		my $current = shift;
 		my $total = shift || 1;
 		my $p = int($current * 100 / $total);
-		if ($p != $last_p) {
+		if ($p < $last_p || $current == 1) {
+			$start_t = time();
+			$last_p = 0;
+		} elsif ($p != $last_p) {
 			my $rate = ($current / (time() - $start_t || 1));
 			my $eta = ($total-$current) / ($rate || 1);
 			printf STDERR ("%5d [%-38s] %-5d %0.1f/s %s\r",$current,"=" x ($p/3)."$p%>", $total, $rate, fmt_time($eta));
@@ -696,16 +698,30 @@
 	}
 
 	my $fake_dir = 1;
+	my $fake_pos = 0;
+	my $last_fake_t = time();
 	sub fakeprogress {
 		return if (! $show_progress);
 		my $current = shift @_;
 
-		my @ind = ('-','\\','|','/','-','\\','|','/', '-');
+		my @ind = ('-','\\','|','/','-','\\','|','/');
+
+		if ($current < $fake_pos) {
+			$start_t = time();
+			$last_fake_t = 0;
+			$fake_dir = 1;
+			$fake_pos = 0;
+		}
 
-		$last_p += $fake_dir;
-		$fake_dir = -$fake_dir if ($last_p > 1000 || $last_p < 0);
-		if ($last_p % 10 == 0) {
-			printf STDERR ("%5d / %5s [%-51s]\r",$current,"?"," " x ($last_p/20).$ind[($last_p/20) % $#ind]);
+		if (time()-$last_fake_t >= 1) {
+			$last_fake_t = time();
+			$fake_pos += $fake_dir;
+			$fake_dir = -$fake_dir if ($fake_pos > 38);
+		}
+
+		if ($current % 10 == 0) {
+			my $rate = ($current / (time() - $start_t || 1));
+			printf STDERR ("%5d [%-38s] %0.1f/s\r",$current, " " x $fake_pos .$ind[($current / 10) % 8], $rate);
 		}
 	}
 
@@ -720,66 +736,22 @@
 		my $isis_db = $cfg -> val($database, 'isis_db') || die "$database doesn't have 'isis_db' defined!";
 
 		$import2cp = Text::Iconv->new($config->{isis_codepage},$codepage);
-		my $db = OpenIsis::open( $isis_db );
+		my $db = new Biblio::Isis( isisdb => $isis_db );
 
-		# check if .txt database for OpenIsis is zero length,
-		# if so, erase it and re-open database
-		sub check_txt_db {
-			my $isis_db = shift || die "need isis database name";
-			my $reopen = 0;
-
-			if (-e $isis_db.".TXT") {
-				print STDERR "WARNING: removing $isis_db.TXT OpenIsis database...\n";
-				unlink $isis_db.".TXT" || warn "FATAL: unlink error on '$isis_db.TXT': $!";
-				$reopen++;
-			}
-			if (-e $isis_db.".PTR") {
-				print STDERR "WARNING: removing $isis_db.PTR OpenIsis database...\n";
-				unlink $isis_db.".PTR" || warn "FATAL: unlink error on '$isis_db.PTR': $!";
-				$reopen++;
-			}
-			return OpenIsis::open( $isis_db ) if ($reopen);
-		}
-
-		# EOF error
-		if ($db == -1) {
-			$db = check_txt_db($isis_db);
-			if ($db == -1) {
-				print STDERR "FATAL: OpenIsis can't open zero size file $isis_db\n";
-				next;
-			}
-		}
-
-		# OpenIsis::ERR_BADF 
-		if ($db == -4) {
-			print STDERR "FATAL: OpenIsis can't find file $isis_db\n";
-			next;
-		# OpenIsis::ERR_IO
-		} elsif ($db == -5) {
-			print STDERR "FATAL: OpenIsis can't access file $isis_db\n";
-			next;
-		} elsif ($db < 0) {
-			print STDERR "FATAL: OpenIsis unknown error $db with file $isis_db\n";
-			next;
-		}
-
-		my $max_rowid = OpenIsis::maxRowid( $db );
-
-		# if 0 records, try to rease isis .txt database
-		if ($max_rowid == 0) {
-			# force removal of database
-			$db = check_txt_db($isis_db);
-			$max_rowid = OpenIsis::maxRowid( $db );
-		}
+		my $max_rowid = $db->count || die "can't find maxmfn";
+$max_rowid = 200; # XXX
 
 		print STDERR "Reading database: $isis_db [$max_rowid rows]\n";
 
-		my $path = $database;
+		$path = $database;
 
 		for (my $row_id = 1; $row_id <= $max_rowid; $row_id++ ) {
-			my $row = OpenIsis::read( $db, $row_id );
-			if ($row && $row->{mfn}) {
-	
+			my $row = $db->to_hash( $row_id );
+			if ($row) {
+
+				$row->{mfn} = $row_id;
+				$row->{record} = $db->{record};
+
 				progress($row->{mfn}, $max_rowid);
 
 				my $swishpath = $path."#".int($row->{mfn});
@@ -793,10 +765,6 @@
 				}
 			}
 		}
-		# for this to work with current version of OpenIsis (0.9.0)
-		# you might need my patch from
-		# http://www.rot13.org/~dpavlin/projects/openisis-0.9.0-perl_close.diff
-		OpenIsis::close($db);
 		print STDERR "\n";
 
 	} elsif ($type_base eq "excel") {
@@ -860,36 +828,40 @@
 		}
 	} elsif ($type_base eq "marc") {
 
-		require MARC;
+		require MARC::File::USMARC;
 		
 		$import2cp = Text::Iconv->new($config->{marc_codepage},$codepage);
 		my $marc_file = $cfg -> val($database, 'marc_file') || die "$database doesn't have 'marc_file' defined!";
 
 		# optional argument is format
-		my $format = x($config->{marc_format}) || 'usmarc';
-
+		warn "marc_format is no longer used!" if ($config->{marc_format});
 		print STDERR "Reading MARC file '$marc_file'\n";
 
-		my $marc = new MARC;
-		my $nr = $marc->openmarc({
-				file=>$marc_file, format=>$format
-			}) || die "Can't open MARC file '$marc_file' with format '$format'";
+		my $marc = MARC::File::USMARC->in( $marc_file )
+			|| die "Can't open MARC file '$marc_file': ".$MARC::File::ERROR;
 
-		# read MARC file in memory
-		$marc->nextmarc(-1);
+		# count records in MARC file
+		sub marc_count {
+			my $filename = shift || die;
+			my $file = MARC::File::USMARC->in($filename) || die $MARC::File::ERROR;
+			my $count = 0;
+			while ($file->skip()) {
+				$count++;
+			}
+			return $count;
+		}
 
-		my $max_rec = $marc->marc_count();
+		my $count = marc_count($marc_file) || warn "no records in '$marc_file'?";
 
-		for(my $i=1; $i<=$max_rec; $i++) {
+		my $i = 0;
 
-			progress($i,$max_rec);
+		while( my $rec = $marc->next() ) {
 
-			# store value for marc_sf.pm
-			$main::cache->{marc_record} = $i;
+			progress($i++,$count);
 
 			my $swishpath = $database."#".$i;
 
-			if (my $xml = data2xml($type_base,$marc,$add_xml,$cfg,$database)) {
+			if (my $xml = data2xml($type_base,$rec,$add_xml,$cfg,$database)) {
 				$xml = $cp2utf->convert($xml);
 				use bytes;	# as opposed to chars
 				print "Path-Name: $swishpath\n";
@@ -960,7 +932,7 @@
 
 =head1 DESCRIPTION
 
-This command will read ISIS data file using OpenIsis perl module, MARC
+This command will read ISIS data file using IsisDB perl module, MARC
 records using MARC module and optionally Micro$oft Excel files to
 create one XML file for usage with I<SWISH-E> indexer. Dispite it's name,
 this script B<isn't general xml generator> from isis files (isis allready