/[Biblio-Isis]/trunk/lib/Biblio/Isis.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/lib/Biblio/Isis.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

trunk/IsisDB.pm revision 16 by dpavlin, Thu Dec 30 17:16:34 2004 UTC trunk/lib/Biblio/Isis.pm revision 44 by dpavlin, Thu Jul 6 11:02:37 2006 UTC
# Line 1  Line 1 
1  package IsisDB;  package Biblio::Isis;
2  use strict;  use strict;
3    
4  use Carp;  use Carp;
5    use File::Glob qw(:globally :nocase);
6    
7  use Data::Dumper;  use Data::Dumper;
8    
9  BEGIN {  BEGIN {
10          use Exporter ();          use Exporter ();
11          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
12          $VERSION     = 0.06;          $VERSION     = 0.13;
13          @ISA         = qw (Exporter);          @ISA         = qw (Exporter);
14          #Give a hoot don't pollute, do not export more than needed by default          #Give a hoot don't pollute, do not export more than needed by default
15          @EXPORT      = qw ();          @EXPORT      = qw ();
# Line 18  BEGIN { Line 20  BEGIN {
20    
21  =head1 NAME  =head1 NAME
22    
23  IsisDB - Read CDS/ISIS, WinISIS and IsisMarc database  Biblio::Isis - Read CDS/ISIS, WinISIS and IsisMarc database
24    
25  =head1 SYNOPSIS  =head1 SYNOPSIS
26    
27    use IsisDB;    use Biblio::Isis;
28    
29    my $isis = new IsisDB(    my $isis = new Biblio::Isis(
30          isisdb => './cds/cds',          isisdb => './cds/cds',
31    );    );
32    
33    for(my $mfn = 1; $mfn <= $isis->{'maxmfn'}; $mfn++) {    for(my $mfn = 1; $mfn <= $isis->count; $mfn++) {
34          print $isis->to_ascii($mfn),"\n";          print $isis->to_ascii($mfn),"\n";
35    }    }
36    
37  =head1 DESCRIPTION  =head1 DESCRIPTION
38    
39  This module will read ISIS databases created by DOS CDS/ISIS, WinIsis or  This module will read ISIS databases created by DOS CDS/ISIS, WinIsis or
40  IsisMarc. It can be used as perl-only alternative to OpenIsis module.  IsisMarc. It can be used as perl-only alternative to OpenIsis module which
41    seems to depriciate it's old C<XS> bindings for perl.
42    
43  It can create hash values from data in ISIS database (using C<to_hash>),  It can create hash values from data in ISIS database (using C<to_hash>),
44  ASCII dump (using C<to_ascii>) or just hash with field names and packed  ASCII dump (using C<to_ascii>) or just hash with field names and packed
# Line 48  fields which are zero sized will be fill Line 51  fields which are zero sized will be fill
51  It also has support for identifiers (only if ISIS database is created by  It also has support for identifiers (only if ISIS database is created by
52  IsisMarc), see C<to_hash>.  IsisMarc), see C<to_hash>.
53    
54  This will module will always be slower than OpenIsis module which use C  This module will always be slower than OpenIsis module which use C
55  library. However, since it's written in perl, it's platform independent (so  library. However, since it's written in perl, it's platform independent (so
56  you don't need C compiler), and can be easily modified. I hope that it  you don't need C compiler), and can be easily modified. I hope that it
57  creates data structures which are easier to use than ones created by  creates data structures which are easier to use than ones created by
# Line 78  rarely an issue). Line 81  rarely an issue).
81    
82  Open ISIS database  Open ISIS database
83    
84   my $isis = new IsisDB(   my $isis = new Biblio::Isis(
85          isisdb => './cds/cds',          isisdb => './cds/cds',
86          read_fdt => 1,          read_fdt => 1,
87          include_deleted => 1,          include_deleted => 1,
# Line 96  Options are described below: Line 99  Options are described below:
99  =item isisdb  =item isisdb
100    
101  This is full or relative path to ISIS database files which include  This is full or relative path to ISIS database files which include
102  common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.  common prefix of C<.MST>, and C<.XRF> and optionally C<.FDT> (if using
103    C<read_fdt> option) files.
104    
105  In this example it uses C<./cds/cds.MST> and related files.  In this example it uses C<./cds/cds.MST> and related files.
106    
# Line 119  Dump a B<lot> of debugging output. Line 123  Dump a B<lot> of debugging output.
123    
124  =back  =back
125    
 It will also set C<$isis-E<gt>{'maxmfn'}> which is maximum MFN stored in database.  
   
126  =cut  =cut
127    
128  sub new {  sub new {
# Line 134  sub new { Line 136  sub new {
136                  $self->{$v} = {@_}->{$v};                  $self->{$v} = {@_}->{$v};
137          }          }
138    
139            my @isis_files = grep(/\.(FDT|MST|XRF|CNT)$/i,glob($self->{isisdb}."*"));
140    
141            foreach my $f (@isis_files) {
142                    my $ext = $1 if ($f =~ m/\.(\w\w\w)$/);
143                    $self->{lc($ext)."_file"} = $f;
144            }
145    
146            my @must_exist = qw(mst xrf);
147            push @must_exist, "fdt" if ($self->{read_fdt});
148    
149            foreach my $ext (@must_exist) {
150                    unless ($self->{$ext."_file"}) {
151                            carp "missing ",uc($ext)," file in ",$self->{isisdb};
152                            return;
153                    }
154            }
155    
156            print STDERR "## using files: ",join(" ",@isis_files),"\n" if ($self->{debug});
157    
158          # if you want to read .FDT file use read_fdt argument when creating class!          # if you want to read .FDT file use read_fdt argument when creating class!
159          if ({@_}->{read_fdt} && -e $self->{isisdb}.".FDT") {          if ($self->{read_fdt} && -e $self->{fdt_file}) {
160    
161                  # read the $db.FDT file for tags                  # read the $db.FDT file for tags
162                  my $fieldzone=0;                  my $fieldzone=0;
163    
164                  open(fileFDT, $self->{isisdb}.".FDT") || croak "can't read '$self->{isisdb}.FDT': $!";                  open(my $fileFDT, $self->{fdt_file}) || croak "can't read '$self->{fdt_file}': $!";
165                    binmode($fileFDT);
166    
167                  while (<fileFDT>) {                  while (<$fileFDT>) {
168                          chomp;                          chomp;
169                          if ($fieldzone) {                          if ($fieldzone) {
170                                  my $name=substr($_,0,30);                                  my $name=substr($_,0,30);
# Line 159  sub new { Line 181  sub new {
181                          }                          }
182                  }                  }
183                                    
184                  close(fileFDT);                  close($fileFDT);
185          }          }
186    
187          # Get the Maximum MFN from $db.MST          # Get the Maximum MFN from $db.MST
188    
189          open(fileMST,$self->{isisdb}.".MST") || croak "can't read '$self->{isisdb}.MST': $!";          open($self->{'fileMST'}, $self->{mst_file}) || croak "can't open '$self->{mst_file}': $!";
190            binmode($self->{'fileMST'});
191    
192          # MST format:   (* = 32 bit signed)          # MST format:   (* = 32 bit signed)
193          # CTLMFN*       always 0          # CTLMFN*       always 0
# Line 172  sub new { Line 195  sub new {
195          # NXTMFB*       last block allocated to master file          # NXTMFB*       last block allocated to master file
196          # NXTMFP        offset to next available position in last block          # NXTMFP        offset to next available position in last block
197          # MFTYPE        always 0 for user db file (1 for system)          # MFTYPE        always 0 for user db file (1 for system)
198          seek(fileMST,4,0);          seek($self->{'fileMST'},4,0) || croak "can't seek to offset 0 in MST: $!";
199    
200          my $buff;          my $buff;
201    
202          read(fileMST, $buff, 4);          read($self->{'fileMST'}, $buff, 4) || croak "can't read NXTMFN from MST: $!";
203          $self->{'NXTMFN'}=unpack("l",$buff) || carp "NXTNFN is zero";          $self->{'NXTMFN'}=unpack("V",$buff) || croak "NXTNFN is zero";
   
         # save maximum MFN  
         $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;  
   
         close(fileMST);  
204    
205          # Get the index information from $db.CNT          print STDERR Dumper($self),"\n" if ($self->{debug});
     
         open(fileCNT, $self->{isisdb}.".CNT") || croak "can't read '$self->{isisdb}.CNT': $!";  
   
         # There is two 26 Bytes fixed lenght records  
   
         #  0: IDTYPE    BTree type                              16  
         #  2: ORDN      Nodes Order                             16  
         #  4: ORDF      Leafs Order                             16  
         #  6: N         Number of Memory buffers for nodes      16  
         #  8: K         Number of buffers for first level index 16  
         # 10: LIV       Current number of Index Levels          16  
         # 12: POSRX*    Pointer to Root Record in N0x           32  
         # 16: NMAXPOS*  Next Available position in N0x          32  
         # 20: FMAXPOS*  Next available position in L0x          32  
         # 24: ABNORMAL  Formal BTree normality indicator        16  
         # length: 26 bytes  
   
         sub unpack_cnt {  
                 my $self = shift;  
   
                 my @flds = qw(ORDN ORDF N K LIV POSRX NMAXPOS FMAXPOS ABNORMAL);  
   
                 my $buff = shift || return;  
                 my @arr = unpack("ssssssllls", $buff);  
   
                 print STDERR "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});  
   
                 my $IDTYPE = shift @arr;  
                 foreach (@flds) {  
                         $self->{$IDTYPE}->{$_} = abs(shift @arr);  
                 }  
         }  
   
         read(fileCNT, $buff, 26);  
         $self->unpack_cnt($buff);  
206    
207          read(fileCNT, $buff, 26);          # open files for later
208          $self->unpack_cnt($buff);          open($self->{'fileXRF'}, $self->{xrf_file}) || croak "can't open '$self->{xrf_file}': $!";
209            binmode($self->{'fileXRF'});
210    
211            $self ? return $self : return undef;
212    }
213    
214          close(fileCNT);  =head2 count
215    
216          print STDERR Dumper($self),"\n" if ($self->{debug});  Return number of records in database
217    
218          # open files for later    print $isis->count;
         open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";  
219    
220          open($self->{'fileMST'}, $self->{isisdb}.".MST") || croak "can't open '$self->{isisdb}.MST': $!";  =cut
221    
222          $self ? return $self : return undef;  sub count {
223            my $self = shift;
224            return $self->{'NXTMFN'} - 1;
225  }  }
226    
227  =head2 fetch  =head2 fetch
# Line 260  sub fetch { Line 247  sub fetch {
247    
248          # is mfn allready in memory?          # is mfn allready in memory?
249          my $old_mfn = $self->{'current_mfn'} || -1;          my $old_mfn = $self->{'current_mfn'} || -1;
250          return if ($mfn == $old_mfn);          return $self->{record} if ($mfn == $old_mfn);
251    
252          print STDERR "## fetch: $mfn\n" if ($self->{debug});          print STDERR "## fetch: $mfn\n" if ($self->{debug});
253    
254          # XXX check this?          # XXX check this?
255          my $mfnpos=($mfn+int(($mfn-1)/127))*4;          my $mfnpos=($mfn+int(($mfn-1)/127))*4;
256    
257          print STDERR "## seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});          print STDERR "## seeking to $mfnpos in file '$self->{xrf_file}'\n" if ($self->{debug});
258          seek($self->{'fileXRF'},$mfnpos,0);          seek($self->{'fileXRF'},$mfnpos,0);
259    
260          my $buff;          my $buff;
261    
262            # delete old record
263            delete $self->{record};
264    
265          # read XRFMFB abd XRFMFP          # read XRFMFB abd XRFMFP
266          read($self->{'fileXRF'}, $buff, 4);          read($self->{'fileXRF'}, $buff, 4);
267          my $pointer=unpack("l",$buff) || carp "pointer is null";          my $pointer=unpack("V",$buff);
268            if (! $pointer) {
269                    if ($self->{include_deleted}) {
270                            return;
271                    } else {
272                            warn "pointer for MFN $mfn is null\n";
273                            return;
274                    }
275            }
276    
277            # check for logically deleted record
278            if ($pointer & 0x80000000) {
279                    print STDERR "## record $mfn is logically deleted\n" if ($self->{debug});
280                    $self->{deleted} = $mfn;
281    
282                    return unless $self->{include_deleted};
283    
284                    # abs
285                    $pointer = ($pointer ^ 0xffffffff) + 1;
286            }
287    
288          my $XRFMFB = int($pointer/2048);          my $XRFMFB = int($pointer/2048);
289          my $XRFMFP = $pointer - ($XRFMFB*2048);          my $XRFMFP = $pointer - ($XRFMFB*2048);
290    
   
291          # (XRFMFB - 1) * 512 + XRFMFP          # (XRFMFB - 1) * 512 + XRFMFP
292          # why do i have to do XRFMFP % 1024 ?          # why do i have to do XRFMFP % 1024 ?
293    
294          my $blk_off = (($XRFMFB - 1) * 512) + ($XRFMFP % 1024);          my $blk_off = (($XRFMFB - 1) * 512) + ($XRFMFP % 512);
295    
296          print STDERR "## pointer: $pointer XRFMFB: $XRFMFB XRFMFP: $XRFMFP offset: $blk_off\n" if ($self->{'debug'});          print STDERR "## pointer: $pointer XRFMFB: $XRFMFB XRFMFP: $XRFMFP offset: $blk_off\n" if ($self->{'debug'});
297    
298          # Get Record Information          # Get Record Information
299    
300          seek($self->{'fileMST'},$blk_off,0);          seek($self->{'fileMST'},$blk_off,0) || croak "can't seek to $blk_off: $!";
301    
302          read($self->{'fileMST'}, $buff, 4);          read($self->{'fileMST'}, $buff, 4) || croak "can't read 4 bytes at offset $blk_off from MST file: $!";
303          my $value=unpack("l",$buff);          my $value=unpack("V",$buff);
304    
305          print STDERR "## offset for rowid $value is $blk_off (blk $XRFMFB off $XRFMFP)\n" if ($self->{debug});          print STDERR "## offset for rowid $value is $blk_off (blk $XRFMFB off $XRFMFP)\n" if ($self->{debug});
306    
307          if ($value!=$mfn) {          if ($value!=$mfn) {
308                  carp "Error: MFN ".$mfn." not found in MST(".$value.")";                      if ($value == 0) {
309                  #return;                # XXX deleted record?                          print STDERR "## record $mfn is physically deleted\n" if ($self->{debug});
310          }                          $self->{deleted} = $mfn;
311                            return;
312                    }
313    
314  #       $MFRL=$self->Read16($fileMST);                  carp "Error: MFN ".$mfn." not found in MST file, found $value";    
315  #       $MFBWB=$self->Read32($fileMST);                  return;
316  #       $MFBWP=$self->Read16($fileMST);          }
 #       $BASE=$self->Read16($fileMST);  
 #       $NVF=$self->Read16($fileMST);  
 #       $STATUS=$self->Read16($fileMST);  
317    
318          read($self->{'fileMST'}, $buff, 14);          read($self->{'fileMST'}, $buff, 14);
319    
320          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("vVvvvv", $buff);
321    
322          print STDERR "## MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});          print STDERR "## MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});
323    
324          # delete old record          warn "MFRL $MFRL is not even number" unless ($MFRL % 2 == 0);
         delete $self->{record};  
   
         ## FIXME this is a bug  
         if (! $self->{'include_deleted'} && $MFRL < 0) {  
                 print "## logically deleted record $mfn, skipping...\n" if ($self->{debug});  
                 return;  
         }  
325    
326          warn "BASE is not 18+6*NVF" unless ($BASE == 18 + 6 * $NVF);          warn "BASE is not 18+6*NVF" unless ($BASE == 18 + 6 * $NVF);
327    
# Line 337  sub fetch { Line 337  sub fetch {
337    
338          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
339    
340  #               $TAG=$self->Read16($fileMST);                  my ($TAG,$POS,$LEN) = unpack("vvv", substr($buff,$i * 6, 6));
 #               $POS=$self->Read16($fileMST);  
 #               $LEN=$self->Read16($fileMST);  
   
                 my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));  
341    
342                  print STDERR "## TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});                  print STDERR "## TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});
343    
# Line 374  sub fetch { Line 370  sub fetch {
370    
371                  push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);                  push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);
372          }          }
         close(fileMST);  
373    
374          $self->{'current_mfn'} = $mfn;          $self->{'current_mfn'} = $mfn;
375    
376          print Dumper($self),"\n" if ($self->{debug});          print STDERR Dumper($self),"\n" if ($self->{debug});
377    
378          return $self->{'record'};          return $self->{'record'};
379  }  }
380    
381  =head2 to_ascii  =head2 to_ascii
382    
383  Dump ASCII output of record with specified MFN  Returns ASCII output of record with specified MFN
384    
385    print $isis->to_ascii(42);    print $isis->to_ascii(42);
386    
387  It outputs something like this:  This outputs something like this:
388    
389    210   ^aNew York^cNew York University press^dcop. 1988    210   ^aNew York^cNew York University press^dcop. 1988
390    990   2140    990   2140
# Line 406  sub to_ascii { Line 401  sub to_ascii {
401    
402          my $mfn = shift || croak "need MFN";          my $mfn = shift || croak "need MFN";
403    
404          my $rec = $self->fetch($mfn);          my $rec = $self->fetch($mfn) || return;
405    
406          my $out = "0\t$mfn";          my $out = "0\t$mfn";
407    
# Line 426  Read record with specified MFN and conve Line 421  Read record with specified MFN and conve
421    
422    my $hash = $isis->to_hash($mfn);    my $hash = $isis->to_hash($mfn);
423    
424  It has ability to convert characters (using C<hash_filter> from ISIS  It has ability to convert characters (using C<hash_filter>) from ISIS
425  database before creating structures enabling character re-mapping or quick  database before creating structures enabling character re-mapping or quick
426  fix-up of data.  fix-up of data.
427    
# Line 474  sub to_hash { Line 469  sub to_hash {
469          # init record to include MFN as field 000          # init record to include MFN as field 000
470          my $rec = { '000' => [ $mfn ] };          my $rec = { '000' => [ $mfn ] };
471    
472          my $row = $self->fetch($mfn);          my $row = $self->fetch($mfn) || return;
473    
474          foreach my $k (keys %{$row}) {          foreach my $k (keys %{$row}) {
475                  foreach my $l (@{$row->{$k}}) {                  foreach my $l (@{$row->{$k}}) {
476    
477                          # filter output                          # filter output
478                          $l = $self->{'hash_filter'}->($l) if ($self->{'hash_filter'});                          if ($self->{'hash_filter'}) {
479                                    $l = $self->{'hash_filter'}->($l);
480                                    next unless defined($l);
481                            }
482    
483                          my $val;                          my $val;
484    
485                          # has identifiers?                          # has identifiers?
486                          ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])//);                          ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])\^/\^/);
487    
488                          # has subfields?                          # has subfields?
489                          if ($l =~ m/\^/) {                          if ($l =~ m/\^/) {
# Line 518  sub tag_name { Line 516  sub tag_name {
516          return $self->{'TagName'}->{$tag} || $tag;          return $self->{'TagName'}->{$tag} || $tag;
517  }  }
518    
519    
520    =head2 read_cnt
521    
522    Read content of C<.CNT> file and return hash containing it.
523    
524      print Dumper($isis->read_cnt);
525    
526    This function is not used by module (C<.CNT> files are not required for this
527    module to work), but it can be useful to examine your index (while debugging
528    for example).
529    
530    =cut
531    
532    sub read_cnt  {
533            my $self = shift;
534    
535            croak "missing CNT file in ",$self->{isisdb} unless ($self->{cnt_file});
536    
537            # Get the index information from $db.CNT
538      
539            open(my $fileCNT, $self->{cnt_file}) || croak "can't read '$self->{cnt_file}': $!";
540            binmode($fileCNT);
541    
542            my $buff;
543    
544            read($fileCNT, $buff, 26) || croak "can't read first table from CNT: $!";
545            $self->unpack_cnt($buff);
546    
547            read($fileCNT, $buff, 26) || croak "can't read second table from CNT: $!";
548            $self->unpack_cnt($buff);
549    
550            close($fileCNT);
551    
552            return $self->{cnt};
553    }
554    
555    =head2 unpack_cnt
556    
557    Unpack one of two 26 bytes fixed length record in C<.CNT> file.
558    
559    Here is definition of record:
560    
561     off key        description                             size
562      0: IDTYPE     BTree type                              s
563      2: ORDN       Nodes Order                             s
564      4: ORDF       Leafs Order                             s
565      6: N          Number of Memory buffers for nodes      s
566      8: K          Number of buffers for first level index s
567     10: LIV        Current number of Index Levels          s
568     12: POSRX      Pointer to Root Record in N0x           l
569     16: NMAXPOS    Next Available position in N0x          l
570     20: FMAXPOS    Next available position in L0x          l
571     24: ABNORMAL   Formal BTree normality indicator        s
572     length: 26 bytes
573    
574    This will fill C<$self> object under C<cnt> with hash. It's used by C<read_cnt>.
575    
576    =cut
577    
578    sub unpack_cnt {
579            my $self = shift;
580    
581            my @flds = qw(ORDN ORDF N K LIV POSRX NMAXPOS FMAXPOS ABNORMAL);
582    
583            my $buff = shift || return;
584            my @arr = unpack("vvvvvvVVVv", $buff);
585    
586            print STDERR "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});
587    
588            my $IDTYPE = shift @arr;
589            foreach (@flds) {
590                    $self->{cnt}->{$IDTYPE}->{$_} = abs(shift @arr);
591            }
592    }
593    
594  1;  1;
595    
596  =head1 BUGS  =head1 BUGS
597    
598  This module has been very lightly tested. Use with caution and report bugs.  Some parts of CDS/ISIS documentation are not detailed enough to exmplain
599    some variations in input databases which has been tested with this module.
600    When I was in doubt, I assumed that OpenIsis's implementation was right
601    (except for obvious bugs).
602    
603    However, every effort has been made to test this module with as much
604    databases (and programs that create them) as possible.
605    
606    I would be very greatful for success or failure reports about usage of this
607    module with databases from programs other than WinIsis and IsisMarc. I had
608    tested this against ouput of one C<isis.dll>-based application, but I don't
609    know any details about it's version.
610    
611  =head1 AUTHOR  =head1 AUTHOR
612    
Legend:
Removed from v.16  
changed lines
  Added in v.44
  ViewVC Help
Powered by ViewVC 1.1.26