/[Biblio-Isis]/trunk/lib/Biblio/Isis.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/lib/Biblio/Isis.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 8 by dpavlin, Wed Dec 29 15:17:59 2004 UTC revision 16 by dpavlin, Thu Dec 30 17:16:34 2004 UTC
# Line 7  use Data::Dumper; Line 7  use Data::Dumper;
7  BEGIN {  BEGIN {
8          use Exporter ();          use Exporter ();
9          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
10          $VERSION     = 0.02;          $VERSION     = 0.06;
11          @ISA         = qw (Exporter);          @ISA         = qw (Exporter);
12          #Give a hoot don't pollute, do not export more than needed by default          #Give a hoot don't pollute, do not export more than needed by default
13          @EXPORT      = qw ();          @EXPORT      = qw ();
# Line 18  BEGIN { Line 18  BEGIN {
18    
19  =head1 NAME  =head1 NAME
20    
21  IsisDB - Read CDS/ISIS database  IsisDB - Read CDS/ISIS, WinISIS and IsisMarc database
22    
23  =head1 SYNOPSIS  =head1 SYNOPSIS
24    
25    use IsisDB    use IsisDB;
26    
27    my $isis = new IsisDB(    my $isis = new IsisDB(
28          isisdb => './cds/cds',          isisdb => './cds/cds',
29    );    );
30    
31      for(my $mfn = 1; $mfn <= $isis->{'maxmfn'}; $mfn++) {
32            print $isis->to_ascii($mfn),"\n";
33      }
34    
35  =head1 DESCRIPTION  =head1 DESCRIPTION
36    
37  This module will read CDS/ISIS databases and create hash values out of it.  This module will read ISIS databases created by DOS CDS/ISIS, WinIsis or
38  It can be used as perl-only alternative to OpenIsis module.  IsisMarc. It can be used as perl-only alternative to OpenIsis module.
39    
40    It can create hash values from data in ISIS database (using C<to_hash>),
41    ASCII dump (using C<to_ascii>) or just hash with field names and packed
42    values (like C<^asomething^belse>).
43    
44    Unique feature of this module is ability to C<include_deleted> records.
45    It will also skip zero sized fields (OpenIsis has a bug in XS bindings, so
46    fields which are zero sized will be filled with random junk from memory).
47    
48    It also has support for identifiers (only if ISIS database is created by
49    IsisMarc), see C<to_hash>.
50    
51    This will module will always be slower than OpenIsis module which use C
52    library. However, since it's written in perl, it's platform independent (so
53    you don't need C compiler), and can be easily modified. I hope that it
54    creates data structures which are easier to use than ones created by
55    OpenIsis, so reduced time in other parts of the code should compensate for
56    slower performance of this module (speed of reading ISIS database is
57    rarely an issue).
58    
59  =head1 METHODS  =head1 METHODS
60    
# Line 50  It can be used as perl-only alternative Line 74  It can be used as perl-only alternative
74  # some binary reads  # some binary reads
75  #  #
76    
 sub Read32 {  
         my $self = shift;  
   
         my $f = shift || die "Read32 needs file handle";  
         read($$f,$b,4) || die "can't read 4 bytes from $$f from position ".tell($f);  
         return unpack("l",$b);  
 }  
   
77  =head2 new  =head2 new
78    
79  Open CDS/ISIS database  Open ISIS database
80    
81   my $isis = new IsisDB(   my $isis = new IsisDB(
82          isisdb => './cds/cds',          isisdb => './cds/cds',
83          read_fdt => 1,          read_fdt => 1,
84            include_deleted => 1,
85            hash_filter => sub {
86                    my $v = shift;
87                    $v =~ s#foo#bar#g;
88            },
89          debug => 1,          debug => 1,
90   );   );
91    
# Line 74  Options are described below: Line 95  Options are described below:
95    
96  =item isisdb  =item isisdb
97    
98  Prefix path to CDS/ISIS. It should contain full or relative path to database  This is full or relative path to ISIS database files which include
99  and common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.  common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.
100    
101    In this example it uses C<./cds/cds.MST> and related files.
102    
103  =item read_fdt  =item read_fdt
104    
105  Boolean flag to specify if field definition table should be read. It's off  Boolean flag to specify if field definition table should be read. It's off
106  by default.  by default.
107    
108    =item include_deleted
109    
110    Don't skip logically deleted records in ISIS.
111    
112    =item hash_filter
113    
114    Filter code ref which will be used before data is converted to hash.
115    
116  =item debug  =item debug
117    
118  Dump a C<lot> of debugging output.  Dump a B<lot> of debugging output.
119    
120  =back  =back
121    
# Line 97  sub new { Line 128  sub new {
128          my $self = {};          my $self = {};
129          bless($self, $class);          bless($self, $class);
130    
131          $self->{isisdb} = {@_}->{isisdb} || croak "new needs database name as argument!";          croak "new needs database name (isisdb) as argument!" unless ({@_}->{isisdb});
132    
133          $self->{debug} = {@_}->{debug};          foreach my $v (qw{isisdb debug include_deleted hash_filter}) {
134                    $self->{$v} = {@_}->{$v};
135            }
136    
137          # if you want to read .FDT file use read_fdt argument when creating class!          # if you want to read .FDT file use read_fdt argument when creating class!
138          if ({@_}->{read_fdt} && -e $self->{isisdb}.".FDT") {          if ({@_}->{read_fdt} && -e $self->{isisdb}.".FDT") {
# Line 140  sub new { Line 173  sub new {
173          # NXTMFP        offset to next available position in last block          # NXTMFP        offset to next available position in last block
174          # MFTYPE        always 0 for user db file (1 for system)          # MFTYPE        always 0 for user db file (1 for system)
175          seek(fileMST,4,0);          seek(fileMST,4,0);
176          $self->{'NXTMFN'}=$self->Read32(\*fileMST) || carp "NXTNFN is zero";  
177            my $buff;
178    
179            read(fileMST, $buff, 4);
180            $self->{'NXTMFN'}=unpack("l",$buff) || carp "NXTNFN is zero";
181    
182          # save maximum MFN          # save maximum MFN
183          $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;          $self->{'maxmfn'} = $self->{'NXTMFN'} - 1;
# Line 173  sub new { Line 210  sub new {
210                  my $buff = shift || return;                  my $buff = shift || return;
211                  my @arr = unpack("ssssssllls", $buff);                  my @arr = unpack("ssssssllls", $buff);
212    
213                    print STDERR "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});
214    
215                  my $IDTYPE = shift @arr;                  my $IDTYPE = shift @arr;
216                  foreach (@flds) {                  foreach (@flds) {
217                          $self->{$IDTYPE}->{$_} = abs(shift @arr);                          $self->{$IDTYPE}->{$_} = abs(shift @arr);
218                  }                  }
219          }          }
220    
         my $buff;  
221          read(fileCNT, $buff, 26);          read(fileCNT, $buff, 26);
222          $self->unpack_cnt($buff);          $self->unpack_cnt($buff);
223    
# Line 189  sub new { Line 227  sub new {
227    
228          close(fileCNT);          close(fileCNT);
229    
230          print Dumper($self) if ($self->{debug});          print STDERR Dumper($self),"\n" if ($self->{debug});
231    
232          # open files for later          # open files for later
233          open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";          open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";
# Line 206  Read record with selected MFN Line 244  Read record with selected MFN
244    my $rec = $isis->fetch(55);    my $rec = $isis->fetch(55);
245    
246  Returns hash with keys which are field names and values are unpacked values  Returns hash with keys which are field names and values are unpacked values
247  for that field.  for that field like this:
248    
249      $rec = {
250        '210' => [ '^aNew York^cNew York University press^dcop. 1988' ],
251        '990' => [ '2140', '88', 'HAY' ],
252      };
253    
254  =cut  =cut
255    
# Line 215  sub fetch { Line 258  sub fetch {
258    
259          my $mfn = shift || croak "fetch needs MFN as argument!";          my $mfn = shift || croak "fetch needs MFN as argument!";
260    
261          print "fetch: $mfn\n" if ($self->{debug});          # is mfn allready in memory?
262            my $old_mfn = $self->{'current_mfn'} || -1;
263            return if ($mfn == $old_mfn);
264    
265            print STDERR "## fetch: $mfn\n" if ($self->{debug});
266    
267          # XXX check this?          # XXX check this?
268          my $mfnpos=($mfn+int(($mfn-1)/127))*4;          my $mfnpos=($mfn+int(($mfn-1)/127))*4;
269    
270          print "seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});          print STDERR "## seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});
271          seek($self->{'fileXRF'},$mfnpos,0);          seek($self->{'fileXRF'},$mfnpos,0);
272    
273            my $buff;
274    
275          # read XRFMFB abd XRFMFP          # read XRFMFB abd XRFMFP
276          my $pointer=$self->Read32(\*{$self->{'fileXRF'}});          read($self->{'fileXRF'}, $buff, 4);
277            my $pointer=unpack("l",$buff) || carp "pointer is null";
278    
279          my $XRFMFB = int($pointer/2048);          my $XRFMFB = int($pointer/2048);
280          my $XRFMFP = $pointer - ($XRFMFB*2048);          my $XRFMFP = $pointer - ($XRFMFB*2048);
281    
         print "XRFMFB: $XRFMFB XRFMFP: $XRFMFP\n" if ($self->{debug});  
282    
283          # XXX fix this to be more readable!!          # (XRFMFB - 1) * 512 + XRFMFP
284          # e.g. (XRFMFB - 1) * 512 + XRFMFP          # why do i have to do XRFMFP % 1024 ?
285    
286          my $offset = $pointer;          my $blk_off = (($XRFMFB - 1) * 512) + ($XRFMFP % 1024);
         my $offset2=int($offset/2048)-1;  
         my $offset22=int($offset/4096);  
         my $offset3=$offset-($offset22*4096);  
         if ($offset3>512) {  
                 $offset3=$offset3-2048;  
         }  
         my $offset4=($offset2*512)+$offset3;  
287    
288          print "$offset - $offset2 - $offset3 - $offset4\n" if ($self->{debug});          print STDERR "## pointer: $pointer XRFMFB: $XRFMFB XRFMFP: $XRFMFP offset: $blk_off\n" if ($self->{'debug'});
289    
290          # Get Record Information          # Get Record Information
291    
292          seek($self->{'fileMST'},$offset4,0);          seek($self->{'fileMST'},$blk_off,0);
293    
294          my $value=$self->Read32(\*{$self->{'fileMST'}});          read($self->{'fileMST'}, $buff, 4);
295            my $value=unpack("l",$buff);
296    
297            print STDERR "## offset for rowid $value is $blk_off (blk $XRFMFB off $XRFMFP)\n" if ($self->{debug});
298    
299          if ($value!=$mfn) {          if ($value!=$mfn) {
300  print ("Error: The MFN:".$mfn." is not found in MST(".$value.")");                      carp "Error: MFN ".$mfn." not found in MST(".$value.")";    
301                  return -1;      # XXX deleted record?                  #return;                # XXX deleted record?
302          }          }
303    
304  #       $MFRL=$self->Read16($fileMST);  #       $MFRL=$self->Read16($fileMST);
# Line 263  print ("Error: The MFN:".$mfn." is not f Line 308  print ("Error: The MFN:".$mfn." is not f
308  #       $NVF=$self->Read16($fileMST);  #       $NVF=$self->Read16($fileMST);
309  #       $STATUS=$self->Read16($fileMST);  #       $STATUS=$self->Read16($fileMST);
310    
         my $buff;  
311          read($self->{'fileMST'}, $buff, 14);          read($self->{'fileMST'}, $buff, 14);
312    
313          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);
314    
315          print "MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});          print STDERR "## MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});
316    
317            # delete old record
318            delete $self->{record};
319    
320            ## FIXME this is a bug
321            if (! $self->{'include_deleted'} && $MFRL < 0) {
322                    print "## logically deleted record $mfn, skipping...\n" if ($self->{debug});
323                    return;
324            }
325    
326            warn "BASE is not 18+6*NVF" unless ($BASE == 18 + 6 * $NVF);
327    
328          # Get Directory Format          # Get Directory Format
329    
# Line 278  print ("Error: The MFN:".$mfn." is not f Line 333  print ("Error: The MFN:".$mfn." is not f
333    
334          read($self->{'fileMST'}, $buff, 6 * $NVF);          read($self->{'fileMST'}, $buff, 6 * $NVF);
335    
336          my $fld_len = 0;          my $rec_len = 0;
337    
338          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
339    
# Line 288  print ("Error: The MFN:".$mfn." is not f Line 343  print ("Error: The MFN:".$mfn." is not f
343    
344                  my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));                  my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));
345    
346                  print "TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});                  print STDERR "## TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});
347    
348                  # The TAG does not exists in .FDT so we set it to 0.                  # The TAG does not exists in .FDT so we set it to 0.
349                  #                  #
# Line 304  print ("Error: The MFN:".$mfn." is not f Line 359  print ("Error: The MFN:".$mfn." is not f
359                  push @FieldPOS,$POS;                  push @FieldPOS,$POS;
360                  push @FieldLEN,$LEN;                  push @FieldLEN,$LEN;
361    
362                  $fld_len += $LEN;                  $rec_len += $LEN;
363          }          }
364    
365          # Get Variable Fields          # Get Variable Fields
366    
367          delete $self->{record};          read($self->{'fileMST'},$buff,$rec_len);
368    
369          read($self->{'fileMST'},$buff,$fld_len);          print STDERR "## rec_len: $rec_len poc: ",tell($self->{'fileMST'})."\n" if ($self->{debug});
370    
371          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
372                    # skip zero-sized fields
373                    next if ($FieldLEN[$i] == 0);
374    
375                  push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);                  push @{$self->{record}->{$FieldTAG[$i]}}, substr($buff,$FieldPOS[$i],$FieldLEN[$i]);
376          }          }
377          close(fileMST);          close(fileMST);
378    
379          # The record is marked for deletion          $self->{'current_mfn'} = $mfn;
         if ($STATUS==1) {  
                 return -1;  
         }  
380    
381          print Dumper($self) if ($self->{debug});          print Dumper($self),"\n" if ($self->{debug});
382    
383          return $self->{'record'};          return $self->{'record'};
384  }  }
385    
386  =head2 to_ascii  =head2 to_ascii
387    
388  Dump ascii output of selected MFN  Dump ASCII output of record with specified MFN
389    
390    print $isis->to_ascii(55);    print $isis->to_ascii(42);
391    
392    It outputs something like this:
393    
394      210   ^aNew York^cNew York University press^dcop. 1988
395      990   2140
396      990   88
397      990   HAY
398    
399    If C<read_fdt> is specified when calling C<new> it will display field names
400    from C<.FDT> file instead of numeric tags.
401    
402  =cut  =cut
403    
# Line 346  sub to_ascii { Line 411  sub to_ascii {
411          my $out = "0\t$mfn";          my $out = "0\t$mfn";
412    
413          foreach my $f (sort keys %{$rec}) {          foreach my $f (sort keys %{$rec}) {
414                  $out .= "\n$f\t".join("\n$f\t",@{$self->{record}->{$f}});                  my $fn = $self->tag_name($f);
415                    $out .= "\n$fn\t".join("\n$fn\t",@{$self->{record}->{$f}});
416          }          }
417    
418          $out .= "\n";          $out .= "\n";
# Line 354  sub to_ascii { Line 420  sub to_ascii {
420          return $out;          return $out;
421  }  }
422    
423  #  =head2 to_hash
424  # XXX porting from php left-over:  
425  #  Read record with specified MFN and convert it to hash
 # do I *REALLY* need those methods, or should I use  
 # $self->{something} directly?  
 #  
 # Probably direct usage is better!  
 #  
426    
427  sub TagName {    my $hash = $isis->to_hash($mfn);
428    
429    It has ability to convert characters (using C<hash_filter> from ISIS
430    database before creating structures enabling character re-mapping or quick
431    fix-up of data.
432    
433    This function returns hash which is like this:
434    
435      $hash = {
436        '210' => [
437                   {
438                     'c' => 'New York University press',
439                     'a' => 'New York',
440                     'd' => 'cop. 1988'
441                   }
442                 ],
443        '990' => [
444                   '2140',
445                   '88',
446                   'HAY'
447                 ],
448      };
449    
450    You can later use that hash to produce any output from ISIS data.
451    
452    If database is created using IsisMarc, it will also have to special fields
453    which will be used for identifiers, C<i1> and C<i2> like this:
454    
455      '200' => [
456                 {
457                   'i1' => '1',
458                   'i2' => ' '
459                   'a' => 'Goa',
460                   'f' => 'Valdo D\'Arienzo',
461                   'e' => 'tipografie e tipografi nel XVI secolo',
462                 }
463               ],
464    
465    This method will also create additional field C<000> with MFN.
466    
467    =cut
468    
469    sub to_hash {
470          my $self = shift;          my $self = shift;
471          return $self->{TagName};  
472            my $mfn = shift || confess "need mfn!";
473    
474            # init record to include MFN as field 000
475            my $rec = { '000' => [ $mfn ] };
476    
477            my $row = $self->fetch($mfn);
478    
479            foreach my $k (keys %{$row}) {
480                    foreach my $l (@{$row->{$k}}) {
481    
482                            # filter output
483                            $l = $self->{'hash_filter'}->($l) if ($self->{'hash_filter'});
484    
485                            my $val;
486    
487                            # has identifiers?
488                            ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])//);
489    
490                            # has subfields?
491                            if ($l =~ m/\^/) {
492                                    foreach my $t (split(/\^/,$l)) {
493                                            next if (! $t);
494                                            $val->{substr($t,0,1)} = substr($t,1);
495                                    }
496                            } else {
497                                    $val = $l;
498                            }
499    
500                            push @{$rec->{$k}}, $val;
501                    }
502            }
503    
504            return $rec;
505  }  }
506    
507  sub NextMFN {  =head2 tag_name
508    
509    Return name of selected tag
510    
511     print $isis->tag_name('200');
512    
513    =cut
514    
515    sub tag_name {
516          my $self = shift;          my $self = shift;
517          return $self->{NXTMFN};          my $tag = shift || return;
518            return $self->{'TagName'}->{$tag} || $tag;
519  }  }
520    
521  1;  1;
# Line 386  This module has been very lightly tested Line 531  This module has been very lightly tested
531          dpavlin@rot13.org          dpavlin@rot13.org
532          http://www.rot13.org/~dpavlin/          http://www.rot13.org/~dpavlin/
533    
534  This module is based heavily on code from LIBISIS.PHP - Library to read ISIS files V0.1.1  This module is based heavily on code from C<LIBISIS.PHP> library to read ISIS files V0.1.1
535  written in php and (c) 2000 Franck Martin - <franck@sopac.org> released under LGPL.  written in php and (c) 2000 Franck Martin <franck@sopac.org> and released under LGPL.
536    
537  =head1 COPYRIGHT  =head1 COPYRIGHT
538    
# Line 400  LICENSE file included with this module. Line 545  LICENSE file included with this module.
545    
546  =head1 SEE ALSO  =head1 SEE ALSO
547    
548  L<http://www.openisis.org|OpenIsis>, perl(1).  OpenIsis web site L<http://www.openisis.org>
549    
550    perl4lib site L<http://perl4lib.perl.org>
551    
Legend:
Removed from v.8  
changed lines
  Added in v.16
  ViewVC Help
Powered by ViewVC 1.1.26