--- branches/tehnika/all2xml.pl 2004/04/18 01:03:27 321 +++ branches/tehnika/all2xml.pl 2005/01/16 18:57:15 633 @@ -1,7 +1,7 @@ #!/usr/bin/perl -w use strict; -use OpenIsis; +use Biblio::Isis; use Getopt::Std; use Data::Dumper; use XML::Simple; @@ -139,12 +139,14 @@ } else { print STDERR "WARNING: field '$field' doesn't have 'name' attribute!"; } + if ($field_name) { + $field_name = x($field_name); if (! $last_field_name) { - $last_field_name = x($field_name); + $last_field_name = $field_name; return $last_field_name; } elsif ($field_name ne $last_field_name) { - $last_field_name = x($field_name); + $last_field_name = $field_name; return $last_field_name; } } @@ -637,6 +639,10 @@ my $lookup_file = $cfg -> val($database, 'lookup_newfile'); # optional if ($lookup_file) { #tie %lhash, 'GDBM_File', $lookup_file, &GDBM_NEWDB, 0644; + if (! -e $lookup_file) { + open(LOOKUP, "> $lookup_file") || die "can't create $lookup_file': $!"; + close(LOOKUP); + } tie %lhash, 'TDB_File', $lookup_file, TDB_CLEAR_IF_FIRST, O_RDWR, 0644; print STDERR "creating lookup file '$lookup_file'\n"; # delete memory cache for lookup file @@ -659,30 +665,62 @@ $config=XMLin("./import_xml/$type.xml", ForceArray => [ $type2tag{$type_base}, 'config', 'format' ], ForceContent => 1 ); + # helper for progress bar + sub fmt_time { + my $t = shift || 0; + my $out = ""; + + my ($ss,$mm,$hh) = gmtime($t); + $out .= "${hh}h" if ($hh); + $out .= sprintf("%02d:%02d", $mm,$ss); + $out .= " " if ($hh == 0); + return $out; + } + # output current progress indicator my $last_p = 0; + my $start_t = time(); sub progress { return if (! $show_progress); my $current = shift; my $total = shift || 1; my $p = int($current * 100 / $total); - if ($p != $last_p) { - printf STDERR ("%5d / %5d [%-51s] %-2d %% \r",$current,$total,"=" x ($p/2).">", $p ); + if ($p < $last_p || $current == 1) { + $start_t = time(); + $last_p = 0; + } elsif ($p != $last_p) { + my $rate = ($current / (time() - $start_t || 1)); + my $eta = ($total-$current) / ($rate || 1); + printf STDERR ("%5d [%-38s] %-5d %0.1f/s %s\r",$current,"=" x ($p/3)."$p%>", $total, $rate, fmt_time($eta)); $last_p = $p; } } my $fake_dir = 1; + my $fake_pos = 0; + my $last_fake_t = time(); sub fakeprogress { return if (! $show_progress); my $current = shift @_; - my @ind = ('-','\\','|','/','-','\\','|','/', '-'); + my @ind = ('-','\\','|','/','-','\\','|','/'); + + if ($current < $fake_pos) { + $start_t = time(); + $last_fake_t = 0; + $fake_dir = 1; + $fake_pos = 0; + } + + if (time()-$last_fake_t >= 1) { + $last_fake_t = time(); + $fake_pos += $fake_dir; + $fake_dir = -$fake_dir if ($fake_pos > 38); + } - $last_p += $fake_dir; - $fake_dir = -$fake_dir if ($last_p > 1000 || $last_p < 0); - if ($last_p % 10 == 0) { - printf STDERR ("%5d / %5s [%-51s]\r",$current,"?"," " x ($last_p/20).$ind[($last_p/20) % $#ind]); + if ($current % 10 == 0) { + my $rate = ($current / (time() - $start_t || 1)); + printf STDERR ("%5d [%-38s] %0.1f/s\r",$current, " " x $fake_pos .$ind[($current / 10) % 8], $rate); } } @@ -697,66 +735,21 @@ my $isis_db = $cfg -> val($database, 'isis_db') || die "$database doesn't have 'isis_db' defined!"; $import2cp = Text::Iconv->new($config->{isis_codepage},$codepage); - my $db = OpenIsis::open( $isis_db ); + my $db = new Biblio::Isis( isisdb => $isis_db ); - # check if .txt database for OpenIsis is zero length, - # if so, erase it and re-open database - sub check_txt_db { - my $isis_db = shift || die "need isis database name"; - my $reopen = 0; - - if (-e $isis_db.".TXT") { - print STDERR "WARNING: removing $isis_db.TXT OpenIsis database...\n"; - unlink $isis_db.".TXT" || warn "FATAL: unlink error on '$isis_db.TXT': $!"; - $reopen++; - } - if (-e $isis_db.".PTR") { - print STDERR "WARNING: removing $isis_db.PTR OpenIsis database...\n"; - unlink $isis_db.".PTR" || warn "FATAL: unlink error on '$isis_db.PTR': $!"; - $reopen++; - } - return OpenIsis::open( $isis_db ) if ($reopen); - } - - # EOF error - if ($db == -1) { - $db = check_txt_db($isis_db); - if ($db == -1) { - print STDERR "FATAL: OpenIsis can't open zero size file $isis_db\n"; - next; - } - } - - # OpenIsis::ERR_BADF - if ($db == -4) { - print STDERR "FATAL: OpenIsis can't find file $isis_db\n"; - next; - # OpenIsis::ERR_IO - } elsif ($db == -5) { - print STDERR "FATAL: OpenIsis can't access file $isis_db\n"; - next; - } elsif ($db < 0) { - print STDERR "FATAL: OpenIsis unknown error $db with file $isis_db\n"; - next; - } - - my $max_rowid = OpenIsis::maxRowid( $db ); - - # if 0 records, try to rease isis .txt database - if ($max_rowid == 0) { - # force removal of database - $db = check_txt_db($isis_db); - $max_rowid = OpenIsis::maxRowid( $db ); - } + my $max_rowid = $db->count || die "can't find maxmfn"; print STDERR "Reading database: $isis_db [$max_rowid rows]\n"; my $path = $database; for (my $row_id = 1; $row_id <= $max_rowid; $row_id++ ) { - my $row = OpenIsis::read( $db, $row_id ); - if ($row && $row->{mfn}) { - + my $row = $db->to_hash( $row_id ); + if ($row) { + + $row->{mfn} = $row_id; + $row->{record} = $db->{record}; + progress($row->{mfn}, $max_rowid); my $swishpath = $path."#".int($row->{mfn}); @@ -770,10 +763,6 @@ } } } - # for this to work with current version of OpenIsis (0.9.0) - # you might need my patch from - # http://www.rot13.org/~dpavlin/projects/openisis-0.9.0-perl_close.diff - OpenIsis::close($db); print STDERR "\n"; } elsif ($type_base eq "excel") { @@ -807,7 +796,11 @@ for(my $iC = $oWorksheet->{MinCol} ; defined $oWorksheet->{MaxCol} && $iC <= $oWorksheet->{MaxCol} ; $iC++) { my $cell = $oWorksheet->{Cells}[$iR][$iC]; if ($cell) { - $row->{int2col($iC)} = $cell->Value; + # this conversion is a cludge. + # Files from Excell could have + # characters which don't fit into + # destination encoding. + $row->{int2col($iC)} = $utf2cp->convert($cell->Value) || $cell->Value; } } @@ -833,36 +826,40 @@ } } elsif ($type_base eq "marc") { - require MARC; + require MARC::File::USMARC; $import2cp = Text::Iconv->new($config->{marc_codepage},$codepage); my $marc_file = $cfg -> val($database, 'marc_file') || die "$database doesn't have 'marc_file' defined!"; # optional argument is format - my $format = x($config->{marc_format}) || 'usmarc'; - + warn "marc_format is no longer used!" if ($config->{marc_format}); print STDERR "Reading MARC file '$marc_file'\n"; - my $marc = new MARC; - my $nr = $marc->openmarc({ - file=>$marc_file, format=>$format - }) || die "Can't open MARC file '$marc_file' with format '$format'"; + my $marc = MARC::File::USMARC->in( $marc_file ) + || die "Can't open MARC file '$marc_file': ".$MARC::File::ERROR; - # read MARC file in memory - $marc->nextmarc(-1); + # count records in MARC file + sub marc_count { + my $filename = shift || die; + my $file = MARC::File::USMARC->in($filename) || die $MARC::File::ERROR; + my $count = 0; + while ($file->skip()) { + $count++; + } + return $count; + } - my $max_rec = $marc->marc_count(); + my $count = marc_count($marc_file) || warn "no records in '$marc_file'?"; - for(my $i=1; $i<=$max_rec; $i++) { + my $i = 0; - progress($i,$max_rec); + while( my $rec = $marc->next() ) { - # store value for marc_sf.pm - $main::cache->{marc_record} = $i; + progress($i++,$count); my $swishpath = $database."#".$i; - if (my $xml = data2xml($type_base,$marc,$add_xml,$cfg,$database)) { + if (my $xml = data2xml($type_base,$rec,$add_xml,$cfg,$database)) { $xml = $cp2utf->convert($xml); use bytes; # as opposed to chars print "Path-Name: $swishpath\n"; @@ -933,7 +930,7 @@ =head1 DESCRIPTION -This command will read ISIS data file using OpenIsis perl module, MARC +This command will read ISIS data file using IsisDB perl module, MARC records using MARC module and optionally Micro$oft Excel files to create one XML file for usage with I indexer. Dispite it's name, this script B from isis files (isis allready