/[Biblio-Isis]/trunk/lib/Biblio/Isis.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/lib/Biblio/Isis.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 12 by dpavlin, Wed Dec 29 20:10:11 2004 UTC revision 16 by dpavlin, Thu Dec 30 17:16:34 2004 UTC
# Line 7  use Data::Dumper; Line 7  use Data::Dumper;
7  BEGIN {  BEGIN {
8          use Exporter ();          use Exporter ();
9          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);          use vars qw ($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
10          $VERSION     = 0.04;          $VERSION     = 0.06;
11          @ISA         = qw (Exporter);          @ISA         = qw (Exporter);
12          #Give a hoot don't pollute, do not export more than needed by default          #Give a hoot don't pollute, do not export more than needed by default
13          @EXPORT      = qw ();          @EXPORT      = qw ();
# Line 18  BEGIN { Line 18  BEGIN {
18    
19  =head1 NAME  =head1 NAME
20    
21  IsisDB - Read CDS/ISIS database  IsisDB - Read CDS/ISIS, WinISIS and IsisMarc database
22    
23  =head1 SYNOPSIS  =head1 SYNOPSIS
24    
# Line 34  IsisDB - Read CDS/ISIS database Line 34  IsisDB - Read CDS/ISIS database
34    
35  =head1 DESCRIPTION  =head1 DESCRIPTION
36    
37  This module will read CDS/ISIS databases and create hash values out of it.  This module will read ISIS databases created by DOS CDS/ISIS, WinIsis or
38  It can be used as perl-only alternative to OpenIsis module.  IsisMarc. It can be used as perl-only alternative to OpenIsis module.
39    
40  This will module will always be slower that OpenIsis module which use C  It can create hash values from data in ISIS database (using C<to_hash>),
41  library. However, since it's written in perl, it's platform independent (so  ASCII dump (using C<to_ascii>) or just hash with field names and packed
42  you don't need C compiler), and can be easily modified.  values (like C<^asomething^belse>).
43    
44  Unique feature of this module is ability to C<include_deleted> records.  Unique feature of this module is ability to C<include_deleted> records.
45  It will also skip zero sized fields (OpenIsis has a bug in XS bindings, so  It will also skip zero sized fields (OpenIsis has a bug in XS bindings, so
46  fields which are zero sized will be filled with random junk from memory).  fields which are zero sized will be filled with random junk from memory).
47    
48    It also has support for identifiers (only if ISIS database is created by
49    IsisMarc), see C<to_hash>.
50    
51    This will module will always be slower than OpenIsis module which use C
52    library. However, since it's written in perl, it's platform independent (so
53    you don't need C compiler), and can be easily modified. I hope that it
54    creates data structures which are easier to use than ones created by
55    OpenIsis, so reduced time in other parts of the code should compensate for
56    slower performance of this module (speed of reading ISIS database is
57    rarely an issue).
58    
59  =head1 METHODS  =head1 METHODS
60    
61  =cut  =cut
# Line 65  fields which are zero sized will be fill Line 76  fields which are zero sized will be fill
76    
77  =head2 new  =head2 new
78    
79  Open CDS/ISIS database  Open ISIS database
80    
81   my $isis = new IsisDB(   my $isis = new IsisDB(
82          isisdb => './cds/cds',          isisdb => './cds/cds',
# Line 84  Options are described below: Line 95  Options are described below:
95    
96  =item isisdb  =item isisdb
97    
98  Prefix path to CDS/ISIS. It should contain full or relative path to database  This is full or relative path to ISIS database files which include
99  and common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.  common prefix of C<.FDT>, C<.MST>, C<.CNT>, C<.XRF> and C<.MST> files.
100    
101    In this example it uses C<./cds/cds.MST> and related files.
102    
103  =item read_fdt  =item read_fdt
104    
# Line 197  sub new { Line 210  sub new {
210                  my $buff = shift || return;                  my $buff = shift || return;
211                  my @arr = unpack("ssssssllls", $buff);                  my @arr = unpack("ssssssllls", $buff);
212    
213                  print "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});                  print STDERR "unpack_cnt: ",join(" ",@arr),"\n" if ($self->{'debug'});
214    
215                  my $IDTYPE = shift @arr;                  my $IDTYPE = shift @arr;
216                  foreach (@flds) {                  foreach (@flds) {
# Line 214  sub new { Line 227  sub new {
227    
228          close(fileCNT);          close(fileCNT);
229    
230          print Dumper($self) if ($self->{debug});          print STDERR Dumper($self),"\n" if ($self->{debug});
231    
232          # open files for later          # open files for later
233          open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";          open($self->{'fileXRF'}, $self->{isisdb}.".XRF") || croak "can't open '$self->{isisdb}.XRF': $!";
# Line 231  Read record with selected MFN Line 244  Read record with selected MFN
244    my $rec = $isis->fetch(55);    my $rec = $isis->fetch(55);
245    
246  Returns hash with keys which are field names and values are unpacked values  Returns hash with keys which are field names and values are unpacked values
247  for that field (like C<^asometing^bsomething else>)  for that field like this:
248    
249      $rec = {
250        '210' => [ '^aNew York^cNew York University press^dcop. 1988' ],
251        '990' => [ '2140', '88', 'HAY' ],
252      };
253    
254  =cut  =cut
255    
# Line 240  sub fetch { Line 258  sub fetch {
258    
259          my $mfn = shift || croak "fetch needs MFN as argument!";          my $mfn = shift || croak "fetch needs MFN as argument!";
260    
261          print "fetch: $mfn\n" if ($self->{debug});          # is mfn allready in memory?
262            my $old_mfn = $self->{'current_mfn'} || -1;
263            return if ($mfn == $old_mfn);
264    
265            print STDERR "## fetch: $mfn\n" if ($self->{debug});
266    
267          # XXX check this?          # XXX check this?
268          my $mfnpos=($mfn+int(($mfn-1)/127))*4;          my $mfnpos=($mfn+int(($mfn-1)/127))*4;
269    
270          print "seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});          print STDERR "## seeking to $mfnpos in file '$self->{isisdb}.XRF'\n" if ($self->{debug});
271          seek($self->{'fileXRF'},$mfnpos,0);          seek($self->{'fileXRF'},$mfnpos,0);
272    
273          my $buff;          my $buff;
# Line 257  sub fetch { Line 279  sub fetch {
279          my $XRFMFB = int($pointer/2048);          my $XRFMFB = int($pointer/2048);
280          my $XRFMFP = $pointer - ($XRFMFB*2048);          my $XRFMFP = $pointer - ($XRFMFB*2048);
281    
         print "XRFMFB: $XRFMFB XRFMFP: $XRFMFP\n" if ($self->{debug});  
282    
283          # XXX fix this to be more readable!!          # (XRFMFB - 1) * 512 + XRFMFP
284          # e.g. (XRFMFB - 1) * 512 + XRFMFP          # why do i have to do XRFMFP % 1024 ?
285    
286          my $offset = $pointer;          my $blk_off = (($XRFMFB - 1) * 512) + ($XRFMFP % 1024);
         my $offset2=int($offset/2048)-1;  
         my $offset22=int($offset/4096);  
         my $offset3=$offset-($offset22*4096);  
         if ($offset3>512) {  
                 $offset3=$offset3-2048;  
         }  
         my $offset4=($offset2*512)+$offset3;  
287    
288          print "$offset - $offset2 - $offset3 - $offset4\n" if ($self->{debug});          print STDERR "## pointer: $pointer XRFMFB: $XRFMFB XRFMFP: $XRFMFP offset: $blk_off\n" if ($self->{'debug'});
289    
290          # Get Record Information          # Get Record Information
291    
292          seek($self->{'fileMST'},$offset4,0);          seek($self->{'fileMST'},$blk_off,0);
293    
294          read($self->{'fileMST'}, $buff, 4);          read($self->{'fileMST'}, $buff, 4);
295          my $value=unpack("l",$buff);          my $value=unpack("l",$buff);
296    
297            print STDERR "## offset for rowid $value is $blk_off (blk $XRFMFB off $XRFMFP)\n" if ($self->{debug});
298    
299          if ($value!=$mfn) {          if ($value!=$mfn) {
300  print ("Error: The MFN:".$mfn." is not found in MST(".$value.")");                      carp "Error: MFN ".$mfn." not found in MST(".$value.")";    
301                  return -1;      # XXX deleted record?                  #return;                # XXX deleted record?
302          }          }
303    
304  #       $MFRL=$self->Read16($fileMST);  #       $MFRL=$self->Read16($fileMST);
# Line 296  print ("Error: The MFN:".$mfn." is not f Line 312  print ("Error: The MFN:".$mfn." is not f
312    
313          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);          my ($MFRL,$MFBWB,$MFBWP,$BASE,$NVF,$STATUS) = unpack("slssss", $buff);
314    
315          print "MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});          print STDERR "## MFRL: $MFRL MFBWB: $MFBWB MFBWP: $MFBWP BASE: $BASE NVF: $NVF STATUS: $STATUS\n" if ($self->{debug});
316    
317          # delete old record          # delete old record
318          delete $self->{record};          delete $self->{record};
319    
320            ## FIXME this is a bug
321          if (! $self->{'include_deleted'} && $MFRL < 0) {          if (! $self->{'include_deleted'} && $MFRL < 0) {
322                  print "## logically deleted record $mfn, skipping...\n" if ($self->{debug});                  print "## logically deleted record $mfn, skipping...\n" if ($self->{debug});
323                  return;                  return;
324          }          }
325    
326            warn "BASE is not 18+6*NVF" unless ($BASE == 18 + 6 * $NVF);
327    
328          # Get Directory Format          # Get Directory Format
329    
330          my @FieldPOS;          my @FieldPOS;
# Line 314  print ("Error: The MFN:".$mfn." is not f Line 333  print ("Error: The MFN:".$mfn." is not f
333    
334          read($self->{'fileMST'}, $buff, 6 * $NVF);          read($self->{'fileMST'}, $buff, 6 * $NVF);
335    
336          my $fld_len = 0;          my $rec_len = 0;
337    
338          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
339    
# Line 324  print ("Error: The MFN:".$mfn." is not f Line 343  print ("Error: The MFN:".$mfn." is not f
343    
344                  my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));                  my ($TAG,$POS,$LEN) = unpack("sss", substr($buff,$i * 6, 6));
345    
346                  print "TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});                  print STDERR "## TAG: $TAG POS: $POS LEN: $LEN\n" if ($self->{debug});
347    
348                  # The TAG does not exists in .FDT so we set it to 0.                  # The TAG does not exists in .FDT so we set it to 0.
349                  #                  #
# Line 340  print ("Error: The MFN:".$mfn." is not f Line 359  print ("Error: The MFN:".$mfn." is not f
359                  push @FieldPOS,$POS;                  push @FieldPOS,$POS;
360                  push @FieldLEN,$LEN;                  push @FieldLEN,$LEN;
361    
362                  $fld_len += $LEN;                  $rec_len += $LEN;
363          }          }
364    
365          # Get Variable Fields          # Get Variable Fields
366    
367          read($self->{'fileMST'},$buff,$fld_len);          read($self->{'fileMST'},$buff,$rec_len);
368    
369            print STDERR "## rec_len: $rec_len poc: ",tell($self->{'fileMST'})."\n" if ($self->{debug});
370    
371          for (my $i = 0 ; $i < $NVF ; $i++) {          for (my $i = 0 ; $i < $NVF ; $i++) {
372                  # skip zero-sized fields                  # skip zero-sized fields
# Line 355  print ("Error: The MFN:".$mfn." is not f Line 376  print ("Error: The MFN:".$mfn." is not f
376          }          }
377          close(fileMST);          close(fileMST);
378    
379          print Dumper($self) if ($self->{debug});          $self->{'current_mfn'} = $mfn;
380    
381            print Dumper($self),"\n" if ($self->{debug});
382    
383          return $self->{'record'};          return $self->{'record'};
384  }  }
385    
386  =head2 to_ascii  =head2 to_ascii
387    
388  Dump ascii output of selected MFN  Dump ASCII output of record with specified MFN
389    
390    print $isis->to_ascii(55);    print $isis->to_ascii(42);
391    
392    It outputs something like this:
393    
394      210   ^aNew York^cNew York University press^dcop. 1988
395      990   2140
396      990   88
397      990   HAY
398    
399    If C<read_fdt> is specified when calling C<new> it will display field names
400    from C<.FDT> file instead of numeric tags.
401    
402  =cut  =cut
403    
# Line 378  sub to_ascii { Line 411  sub to_ascii {
411          my $out = "0\t$mfn";          my $out = "0\t$mfn";
412    
413          foreach my $f (sort keys %{$rec}) {          foreach my $f (sort keys %{$rec}) {
414                  $out .= "\n$f\t".join("\n$f\t",@{$self->{record}->{$f}});                  my $fn = $self->tag_name($f);
415                    $out .= "\n$fn\t".join("\n$fn\t",@{$self->{record}->{$f}});
416          }          }
417    
418          $out .= "\n";          $out .= "\n";
# Line 388  sub to_ascii { Line 422  sub to_ascii {
422    
423  =head2 to_hash  =head2 to_hash
424    
425  Read mfn and convert it to hash  Read record with specified MFN and convert it to hash
426    
427    my $hash = $isis->to_hash($mfn);    my $hash = $isis->to_hash($mfn);
428    
429  It has ability to convert characters (using C<hash_filter> from ISIS  It has ability to convert characters (using C<hash_filter> from ISIS
430  database before creating structures enabling character remapping or quick  database before creating structures enabling character re-mapping or quick
431  fixup of data.  fix-up of data.
432    
433  This function returns hash which is like this:  This function returns hash which is like this:
434    
# Line 413  This function returns hash which is like Line 447  This function returns hash which is like
447               ],               ],
448    };    };
449    
450  You can later use that has to produce any output from ISIS data.  You can later use that hash to produce any output from ISIS data.
451    
452    If database is created using IsisMarc, it will also have to special fields
453    which will be used for identifiers, C<i1> and C<i2> like this:
454    
455      '200' => [
456                 {
457                   'i1' => '1',
458                   'i2' => ' '
459                   'a' => 'Goa',
460                   'f' => 'Valdo D\'Arienzo',
461                   'e' => 'tipografie e tipografi nel XVI secolo',
462                 }
463               ],
464    
465    This method will also create additional field C<000> with MFN.
466    
467  =cut  =cut
468    
# Line 422  sub to_hash { Line 471  sub to_hash {
471    
472          my $mfn = shift || confess "need mfn!";          my $mfn = shift || confess "need mfn!";
473    
474          my $rec;          # init record to include MFN as field 000
475            my $rec = { '000' => [ $mfn ] };
476    
477          my $row = $self->fetch($mfn);          my $row = $self->fetch($mfn);
478    
479          foreach my $k (keys %{$row}) {          foreach my $k (keys %{$row}) {
# Line 431  sub to_hash { Line 482  sub to_hash {
482                          # filter output                          # filter output
483                          $l = $self->{'hash_filter'}->($l) if ($self->{'hash_filter'});                          $l = $self->{'hash_filter'}->($l) if ($self->{'hash_filter'});
484    
                         # has subfields?  
485                          my $val;                          my $val;
486    
487                            # has identifiers?
488                            ($val->{'i1'},$val->{'i2'}) = ($1,$2) if ($l =~ s/^([01 #])([01 #])//);
489    
490                            # has subfields?
491                          if ($l =~ m/\^/) {                          if ($l =~ m/\^/) {
492                                  foreach my $t (split(/\^/,$l)) {                                  foreach my $t (split(/\^/,$l)) {
493                                          next if (! $t);                                          next if (! $t);
# Line 449  sub to_hash { Line 504  sub to_hash {
504          return $rec;          return $rec;
505  }  }
506    
507  #  =head2 tag_name
 # XXX porting from php left-over:  
 #  
 # do I *REALLY* need those methods, or should I use  
 # $self->{something} directly?  
 #  
 # Probably direct usage is better!  
 #  
508    
509  sub TagName {  Return name of selected tag
         my $self = shift;  
         return $self->{TagName};  
 }  
510    
511  sub NextMFN {   print $isis->tag_name('200');
512    
513    =cut
514    
515    sub tag_name {
516          my $self = shift;          my $self = shift;
517          return $self->{NXTMFN};          my $tag = shift || return;
518            return $self->{'TagName'}->{$tag} || $tag;
519  }  }
520    
521  1;  1;
# Line 481  This module has been very lightly tested Line 531  This module has been very lightly tested
531          dpavlin@rot13.org          dpavlin@rot13.org
532          http://www.rot13.org/~dpavlin/          http://www.rot13.org/~dpavlin/
533    
534  This module is based heavily on code from LIBISIS.PHP - Library to read ISIS files V0.1.1  This module is based heavily on code from C<LIBISIS.PHP> library to read ISIS files V0.1.1
535  written in php and (c) 2000 Franck Martin - <franck@sopac.org> released under LGPL.  written in php and (c) 2000 Franck Martin <franck@sopac.org> and released under LGPL.
536    
537  =head1 COPYRIGHT  =head1 COPYRIGHT
538    
# Line 495  LICENSE file included with this module. Line 545  LICENSE file included with this module.
545    
546  =head1 SEE ALSO  =head1 SEE ALSO
547    
548  L<http://www.openisis.org|OpenIsis>, perl(1).  OpenIsis web site L<http://www.openisis.org>
549    
550    perl4lib site L<http://perl4lib.perl.org>
551    
Legend:
Removed from v.12  
changed lines
  Added in v.16
  ViewVC Help
Powered by ViewVC 1.1.26