--- trunk/all2xml.pl	2004/12/31 04:22:49	619
+++ trunk/all2xml.pl	2005/02/17 18:08:14	672
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -w
 
 use strict;
-use IsisDB;
+use Biblio::Isis;
 use Getopt::Std;
 use Data::Dumper;
 use XML::Simple;
@@ -22,7 +22,8 @@
 my $config;
 
 #use index_DBI;		# default DBI module for index
-use index_DBI_cache;	# faster DBI module using memory cache
+#use index_DBI_cache;	# faster DBI module using memory cache
+use index_DBI_filter;	# filter support for indexes
 my $index;
 
 my %opts;
@@ -257,8 +258,6 @@
 			# init vars so that we go into while...
 			($swish,$display) = (1,1);
 
-			# placeholder for all repeatable entries for index
-
 			sub mkformat($$) {
 				my $x = shift || die "mkformat needs tag reference";
 				my $data = shift || return;
@@ -634,6 +633,10 @@
 
 foreach my $database ($cfg->Sections) {
 
+	# save database name in global variable path for later
+	# (need for index filter creation)
+	$path = $database;
+
 	my $type = lc($cfg -> val($database, 'type')) || die "$database doesn't have 'type' defined";
 	my $add_xml = $cfg -> val($database, 'xml');	# optional
 
@@ -684,10 +687,13 @@
 	my $start_t = time();
 	sub progress {
 		return if (! $show_progress);
-		my $current = shift || 1;
+		my $current = shift;
 		my $total = shift || 1;
 		my $p = int($current * 100 / $total);
-		if ($p != $last_p) {
+		if ($p < $last_p || $current == 1) {
+			$start_t = time();
+			$last_p = 0;
+		} elsif ($p != $last_p) {
 			my $rate = ($current / (time() - $start_t || 1));
 			my $eta = ($total-$current) / ($rate || 1);
 			printf STDERR ("%5d [%-38s] %-5d %0.1f/s %s\r",$current,"=" x ($p/3)."$p%>", $total, $rate, fmt_time($eta));
@@ -696,16 +702,30 @@
 	}
 
 	my $fake_dir = 1;
+	my $fake_pos = 0;
+	my $last_fake_t = time();
 	sub fakeprogress {
 		return if (! $show_progress);
 		my $current = shift @_;
 
-		my @ind = ('-','\\','|','/','-','\\','|','/', '-');
+		my @ind = ('-','\\','|','/','-','\\','|','/');
+
+		if ($current < $fake_pos) {
+			$start_t = time();
+			$last_fake_t = 0;
+			$fake_dir = 1;
+			$fake_pos = 0;
+		}
+
+		if (time()-$last_fake_t >= 1) {
+			$last_fake_t = time();
+			$fake_pos += $fake_dir;
+			$fake_dir = -$fake_dir if ($fake_pos > 38);
+		}
 
-		$last_p += $fake_dir;
-		$fake_dir = -$fake_dir if ($last_p > 1000 || $last_p < 0);
-		if ($last_p % 10 == 0) {
-			printf STDERR ("%5d / %5s [%-51s]\r",$current,"?"," " x ($last_p/20).$ind[($last_p/20) % $#ind]);
+		if ($current % 10 == 0) {
+			my $rate = ($current / (time() - $start_t || 1));
+			printf STDERR ("%5d [%-38s] %0.1f/s\r",$current, " " x $fake_pos .$ind[($current / 10) % 8], $rate);
 		}
 	}
 
@@ -720,13 +740,16 @@
 		my $isis_db = $cfg -> val($database, 'isis_db') || die "$database doesn't have 'isis_db' defined!";
 
 		$import2cp = Text::Iconv->new($config->{isis_codepage},$codepage);
-		my $db = new IsisDB( isisdb => $isis_db );
+		my $db = new Biblio::Isis( isisdb => $isis_db );
 
-		my $max_rowid = $db->{'maxmfn'} || die "can't find maxmfn";
+		my $max_rowid = $db->count if ($db);
 
-		print STDERR "Reading database: $isis_db [$max_rowid rows]\n";
+		if (! $max_rowid) {
+			print STDERR "FATAL: can't read ISIS database: $isis_db, skipping...\n";
+			next;
+		}
 
-		my $path = $database;
+		print STDERR "Reading database: $isis_db [$max_rowid rows]\n";
 
 		for (my $row_id = 1; $row_id <= $max_rowid; $row_id++ ) {
 			my $row = $db->to_hash( $row_id );
@@ -809,44 +832,57 @@
 				print "Document-Type: XML\n\n$xml\n";
 			}
 		}
+
+		print STDERR "\n";
+
 	} elsif ($type_base eq "marc") {
 
-		require MARC;
+		require MARC::File::USMARC;
 		
 		$import2cp = Text::Iconv->new($config->{marc_codepage},$codepage);
 		my $marc_file = $cfg -> val($database, 'marc_file') || die "$database doesn't have 'marc_file' defined!";
 
 		# optional argument is format
-		my $format = x($config->{marc_format}) || 'usmarc';
-
+		warn "marc_format is no longer used!" if ($config->{marc_format});
 		print STDERR "Reading MARC file '$marc_file'\n";
 
-		my $marc = new MARC;
-		my $nr = $marc->openmarc({
-				file=>$marc_file, format=>$format
-			}) || die "Can't open MARC file '$marc_file' with format '$format'";
+		my $marc = MARC::File::USMARC->in( $marc_file );
 
-		# read MARC file in memory
-		$marc->nextmarc(-1);
+		if (! $marc) {
+			print STDERR "FATAL: can't read MARC file: $marc_file, skipping...\n";
+			next;
+		}
 
-		my $max_rec = $marc->marc_count();
+		# count records in MARC file
+		sub marc_count {
+			my $filename = shift || die;
+			my $file = MARC::File::USMARC->in($filename) || return;
+			my $count = 0;
+			while ($file->skip()) {
+				$count++;
+			}
+			return $count;
+		}
 
-		for(my $i=1; $i<=$max_rec; $i++) {
+		my $count = marc_count($marc_file) || warn "no records in '$marc_file'?";
 
-			progress($i,$max_rec);
+		my $i = 1;
 
-			# store value for marc_sf.pm
-			$main::cache->{marc_record} = $i;
+		while( my $rec = $marc->next() ) {
+
+			progress($i,$count);
 
 			my $swishpath = $database."#".$i;
 
-			if (my $xml = data2xml($type_base,$marc,$add_xml,$cfg,$database)) {
+			if (my $xml = data2xml($type_base,$rec,$add_xml,$cfg,$database)) {
 				$xml = $cp2utf->convert($xml);
 				use bytes;	# as opposed to chars
 				print "Path-Name: $swishpath\n";
 				print "Content-Length: ".(length($xml)+1)."\n";
 				print "Document-Type: XML\n\n$xml\n";
 			}
+
+			$i++;
 		}
 
 		print STDERR "\n";
@@ -911,8 +947,8 @@
 
 =head1 DESCRIPTION
 
-This command will read ISIS data file using OpenIsis perl module, MARC
-records using MARC module and optionally Micro$oft Excel files to
+This command will read ISIS data file using Biblio::Isis perl module, MARC
+records using MARC::File module and optionally Micro$oft Excel files to
 create one XML file for usage with I<SWISH-E> indexer. Dispite it's name,
 this script B<isn't general xml generator> from isis files (isis allready
 has something like that). Output of this script is tailor-made for SWISH-E.