/[webpac]/trunk2/lib/WebPAC.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk2/lib/WebPAC.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 357 by dpavlin, Wed Jun 16 13:39:17 2004 UTC revision 368 by dpavlin, Thu Jun 17 12:27:02 2004 UTC
# Line 1  Line 1 
1  package WebPAC;  package WebPAC;
2    
3    use warnings;
4    use strict;
5    
6  use Carp;  use Carp;
7  use Text::Iconv;  use Text::Iconv;
8  use Config::IniFiles;  use Config::IniFiles;
9    use XML::Simple;
10    
11    use Data::Dumper;
12    
13  =head1 NAME  =head1 NAME
14    
# Line 32  which describes databases to be indexed. Line 38  which describes databases to be indexed.
38    
39  =cut  =cut
40    
41    # mapping between data type and tag which specify
42    # format in XML file
43    my %type2tag = (
44            'isis' => 'isis',
45    #       'excel' => 'column',
46    #       'marc' => 'marc',
47    #       'feed' => 'feed'
48    );
49    
50  sub new {  sub new {
51          my $class = shift;          my $class = shift;
52          my $self = {@_};          my $self = {@_};
# Line 65  sub new { Line 80  sub new {
80    
81          $self->{indexer_config_file} = new Config::IniFiles( -file => $self->{config_file} ) || croak "can't open '$self->{config_file}'";          $self->{indexer_config_file} = new Config::IniFiles( -file => $self->{config_file} ) || croak "can't open '$self->{config_file}'";
82    
83          # read global config parametars          $self->{'utf2cp'} = Text::Iconv->new('UTF-8' ,$self->{'code_page'});
         foreach my $var (qw(  
                         dbi_dbd  
                         dbi_dsn  
                         dbi_user  
                         dbi_passwd  
                         show_progress  
                         my_unac_filter  
                 )) {  
                 $self->{global_config}->{$var} = $self->{global_config_file}->val('global', $var);  
         }  
   
84          return $self;          return $self;
85  }  }
86    
# Line 96  By default, ISIS code page is assumed to Line 100  By default, ISIS code page is assumed to
100  If optional parametar C<limit_mfn> is set, it will read just 500 records  If optional parametar C<limit_mfn> is set, it will read just 500 records
101  from database in example above.  from database in example above.
102    
 Returns number of last record read into memory (size of database, really).  
   
103  C<lookup> argument is an array of lookups to create. Each lookup must have C<key> and  C<lookup> argument is an array of lookups to create. Each lookup must have C<key> and
104  C<val>. Optional parametar C<eval> is perl code to evaluate before storing  C<val>. Optional parametar C<eval> is perl code to evaluate before storing
105  value in index.  value in index.
# Line 109  value in index. Line 111  value in index.
111      'val' => 'v900' },      'val' => 'v900' },
112   ]   ]
113    
114    Returns number of last record read into memory (size of database, really).
115    
116  =cut  =cut
117    
118  sub open_isis {  sub open_isis {
# Line 125  sub open_isis { Line 129  sub open_isis {
129          # create Text::Iconv object          # create Text::Iconv object
130          my $cp = Text::Iconv->new($code_page,$self->{'code_page'});          my $cp = Text::Iconv->new($code_page,$self->{'code_page'});
131    
132            print STDERR "reading ISIS database '",$arg->{'filename'},"'\n" if ($self->{'debug'});
133    
134          my $isis_db = OpenIsis::open($arg->{'filename'});          my $isis_db = OpenIsis::open($arg->{'filename'});
135    
136          my $maxmfn = OpenIsis::maxRowid( $isis_db ) || 1;          my $maxmfn = OpenIsis::maxRowid( $isis_db ) || 1;
137    
138          $maxmfn = $self->{limit_mfn} if ($self->{limit_mfn});          $maxmfn = $self->{limit_mfn} if ($self->{limit_mfn});
139    
140            print STDERR "processing $maxmfn records...\n" if ($self->{'debug'});
141    
142          # read database          # read database
143          for (my $mfn = 1; $mfn <= $maxmfn; $mfn++) {          for (my $mfn = 1; $mfn <= $maxmfn; $mfn++) {
144    
# Line 163  sub open_isis { Line 171  sub open_isis {
171    
172          }          }
173    
174            $self->{'current_mfn'} = 1;
175    
176          # store max mfn and return it.          # store max mfn and return it.
177          return $self->{'max_mfn'} = $maxmfn;          return $self->{'max_mfn'} = $maxmfn;
178  }  }
179    
180    =head2 fetch_rec
181    
182    Fetch next record from database. It will also display progress bar (once
183    it's implemented, that is).
184    
185     my $rec = $webpac->fetch_rec;
186    
187    =cut
188    
189    sub fetch_rec {
190            my $self = shift;
191    
192            my $mfn = $self->{'current_mfn'}++ || confess "it seems that you didn't load database!";
193    
194            if ($mfn > $self->{'max_mfn'}) {
195                    $self->{'current_mfn'} = $self->{'max_mfn'};
196                    return;
197            }
198    
199            return $self->{'data'}->{$mfn};
200    }
201    
202    =head2 open_import_xml
203    
204    Read file from C<import_xml/> directory and parse it.
205    
206     $webpac->open_import_xml(type => 'isis');
207    
208    =cut
209    
210    sub open_import_xml {
211            my $self = shift;
212    
213            my $arg = {@_};
214            confess "need type to load file from import_xml/" if (! $arg->{'type'});
215    
216            $self->{'type'} = $arg->{'type'};
217    
218            my $type_base = $arg->{'type'};
219            $type_base =~ s/_.*$//g;
220    
221            $self->{'tag'} = $type2tag{$type_base};
222    
223            print STDERR "using type ",$self->{'type'}," tag ",$self->{'tag'},"\n" if ($self->{'debug'});
224    
225            my $f = "./import_xml/".$self->{'type'}.".xml";
226            confess "import_xml file '$f' doesn't exist!" if (! -e "$f");
227    
228            print STDERR "reading '$f'\n" if ($self->{'debug'});
229    
230            $self->{'import_xml'} = XMLin($f,
231                    ForceArray => [ $self->{'tag'}, 'config', 'format' ],
232                    ForceContent => 1
233            );
234    
235            print Dumper($self->{'import_xml'});
236    
237    }
238    
239  =head2 create_lookup  =head2 create_lookup
240    
241  Create lookup from record using lookup definition.  Create lookup from record using lookup definition.
242    
243     $self->create_lookup($rec, @lookups);
244    
245    Called internally by C<open_*> methods.
246    
247  =cut  =cut
248    
249  sub create_lookup {  sub create_lookup {
# Line 201  sub create_lookup { Line 274  sub create_lookup {
274    
275  Returns value from record.  Returns value from record.
276    
277   $self->get_data(\$rec,$f,$sf,$i,\$found);   my $text = $self->get_data(\$rec,$f,$sf,$i,\$found);
278    
279  Arguments are:  Arguments are:
280  record reference C<$rec>,  record reference C<$rec>,
# Line 209  field C<$f>, Line 282  field C<$f>,
282  optional subfiled C<$sf>,  optional subfiled C<$sf>,
283  index for repeatable values C<$i>.  index for repeatable values C<$i>.
284    
285  Optinal variable C<$found> will be incremeted if thre  Optinal variable C<$found> will be incremeted if there
286  is field.  is field.
287    
288  Returns value or empty string.  Returns value or empty string.
# Line 220  sub get_data { Line 293  sub get_data {
293          my $self = shift;          my $self = shift;
294    
295          my ($rec,$f,$sf,$i,$found) = @_;          my ($rec,$f,$sf,$i,$found) = @_;
296    
297          if ($$rec->{$f}) {          if ($$rec->{$f}) {
298                    return '' if (! $$rec->{$f}->[$i]);
299                  if ($sf && $$rec->{$f}->[$i]->{$sf}) {                  if ($sf && $$rec->{$f}->[$i]->{$sf}) {
300                          $$found++ if (defined($$found));                          $$found++ if (defined($$found));
301                          return $$rec->{$f}->[$i]->{$sf};                          return $$rec->{$f}->[$i]->{$sf};
302                  } elsif ($$rec->{$f}->[$i]) {                  } elsif ($$rec->{$f}->[$i]) {
303                          $$found++ if (defined($$found));                          $$found++ if (defined($$found));
304                          return $$rec->{$f}->[$i];                          # it still might have subfield, just
305                            # not specified, so we'll dump all
306                            if ($$rec->{$f}->[$i] =~ /HASH/o) {
307                                    my $out;
308                                    foreach my $k (keys %{$$rec->{$f}->[$i]}) {
309                                            $out .= $$rec->{$f}->[$i]->{$k}." ";
310                                    }
311                                    return $out;
312                            } else {
313                                    return $$rec->{$f}->[$i];
314                            }
315                  }                  }
316          } else {          } else {
317                  return '';                  return '';
# Line 239  Workhourse of all: takes record from in- Line 324  Workhourse of all: takes record from in-
324  strings with placeholders and returns string or array of with substituted  strings with placeholders and returns string or array of with substituted
325  values from record.  values from record.
326    
327   $webpac->fill_in($rec,'v250^a');   my $text = $webpac->fill_in($rec,'v250^a');
328    
329  Optional argument is ordinal number for repeatable fields. By default,  Optional argument is ordinal number for repeatable fields. By default,
330  it's assume to be first repeatable field (fields are perl array, so first  it's assume to be first repeatable field (fields are perl array, so first
331  element is 0).  element is 0).
332  Following example will read second value from repeatable field.  Following example will read second value from repeatable field.
333    
334   $webpac->fill_in($rec,'Title: v250^a',1);   my $text = $webpac->fill_in($rec,'Title: v250^a',1);
335    
336  This function B<does not> perform parsing of format to inteligenty skip  This function B<does not> perform parsing of format to inteligenty skip
337  delimiters before fields which aren't used.  delimiters before fields which aren't used.
# Line 266  sub fill_in { Line 351  sub fill_in {
351    
352          my $found = 0;          my $found = 0;
353    
354            my $eval_code;
355            # remove eval{...} from beginning
356            $eval_code = $1 if ($format =~ s/^eval{([^}]+)}//s);
357    
358          # do actual replacement of placeholders          # do actual replacement of placeholders
359          $format =~ s/v(\d+)(?:\^(\w))*/$self->get_data(\$rec,$1,$2,$i,\$found)/ges;          $format =~ s/v(\d+)(?:\^(\w))*/$self->get_data(\$rec,$1,$2,$i,\$found)/ges;
360    
361          if ($found) {          if ($found) {
362                    if ($eval_code) {
363                            my $eval = $self->fill_in($rec,$eval_code,$i);
364                            return if (! eval $eval);
365                    }
366                  # do we have lookups?                  # do we have lookups?
367                  if ($format =~ /\[[^\[\]]+\]/o) {                  if ($format =~ /\[[^\[\]]+\]/o) {
368                          return $self->lookup($format);                          return $self->lookup($format);
# Line 285  sub fill_in { Line 378  sub fill_in {
378    
379  Perform lookups on format supplied to it.  Perform lookups on format supplied to it.
380    
381   my $txt = $self->lookup('[v900]');   my $text = $self->lookup('[v900]');
382    
383  Lookups can be nested (like C<[d:[a:[v900]]]>).  Lookups can be nested (like C<[d:[a:[v900]]]>).
384    
# Line 298  sub lookup { Line 391  sub lookup {
391    
392          if ($tmp =~ /\[[^\[\]]+\]/o) {          if ($tmp =~ /\[[^\[\]]+\]/o) {
393                  my @in = ( $tmp );                  my @in = ( $tmp );
 #print "##lookup $tmp\n";  
394                  my @out;                  my @out;
395                  while (my $f = shift @in) {                  while (my $f = shift @in) {
396                          if ($f =~ /\[([^\[\]]+)\]/) {                          if ($f =~ /\[([^\[\]]+)\]/) {
397                                  my $k = $1;                                  my $k = $1;
398                                  if ($self->{'lookup'}->{$k}) {                                  if ($self->{'lookup'}->{$k}) {
 #print "## lookup key = $k\n";  
399                                          foreach my $nv (@{$self->{'lookup'}->{$k}}) {                                          foreach my $nv (@{$self->{'lookup'}->{$k}}) {
400                                                  my $tmp2 = $f;                                                  my $tmp2 = $f;
401                                                  $tmp2 =~ s/\[$k\]/$nv/g;                                                  $tmp2 =~ s/\[$k\]/$nv/g;
402                                                  push @in, $tmp2;                                                  push @in, $tmp2;
 #print "## lookup in => $tmp2\n";  
403                                          }                                          }
404                                  } else {                                  } else {
405                                          undef $f;                                          undef $f;
406                                  }                                  }
407                          } elsif ($f) {                          } elsif ($f) {
408                                  push @out, $f;                                  push @out, $f;
 #print "## lookup out => $f\n";  
409                          }                          }
410                  }                  }
411                  return @out;                  return @out;
# Line 331  Perform smart parsing of string, skippin Line 420  Perform smart parsing of string, skippin
420  defined. It can also eval code in format starting with C<eval{...}> and  defined. It can also eval code in format starting with C<eval{...}> and
421  return output or nothing depending on eval code.  return output or nothing depending on eval code.
422    
423   $webpac->parse($rec,'eval{"v901^a" eq "Deskriptor"}descriptor: v250^a', $i);   my $text = $webpac->parse($rec,'eval{"v901^a" eq "Deskriptor"}descriptor: v250^a', $i);
424    
425  =cut  =cut
426    
427  sub parse {  sub parse {
428          my $self = shift;          my $self = shift;
429    
430          my ($rec, $format, $i) = @_;          my ($rec, $format_utf8, $i) = @_;
431    
432            return if (! $format_utf8);
433    
434            confess("need HASH as first argument!") if ($rec !~ /HASH/o);
435            confess("need utf2cp Text::Iconv object!") if (! $self->{'utf2cp'});
436    
437            $i = 0 if (! $i);
438    
439            my $format = $self->{'utf2cp'}->convert($format_utf8) || confess("can't convert '$format_utf8' from UTF-8 to ",$self->{'code_page'});
440    
441          my @out;          my @out;
442    
# Line 346  sub parse { Line 444  sub parse {
444          # remove eval{...} from beginning          # remove eval{...} from beginning
445          $eval_code = $1 if ($format =~ s/^eval{([^}]+)}//s);          $eval_code = $1 if ($format =~ s/^eval{([^}]+)}//s);
446    
447          my $prefix = '';          my $prefix;
448          $prefix = $1 if ($format =~ s/^(.+)(v\d+(?:\^\w)*)/$2/s);          my $all_found=0;
449    
450            while ($format =~ s/^(.*?)v(\d+)(?:\^(\w))*//s) {
451    
452          sub f_sf_del {                  my $del = $1 || '';
453                  my ($self,$rec,$out,$f,$sf,$del,$i) = @_;                  $prefix ||= $del if ($all_found == 0);
454    
455                    my $found = 0;
456                    my $tmp = $self->get_data(\$rec,$2,$3,$i,\$found);
457    
                 my $found=0;  
                 my $tmp = $self->get_data($rec,$f,$sf,$i,\$found);  
458                  if ($found) {                  if ($found) {
459                          push @{$$out}, $tmp;                          push @out, $del;
460                          push @{$$out}, $del;                          push @out, $tmp;
461                            $all_found += $found;
462                  }                  }
                 return '';  
463          }          }
464    
465          #$format =~ s/(.*)v(\d+)(?:\^(\w))*/f_sf_del($self,\$rec,\@out,$2,$3,$1,$i/ges;          return if (! $all_found);
466    
467            my $out = join('',@out) . $format;
468    
469            # add prefix if not there
470            $out = $prefix . $out if ($out !~ m/^\Q$prefix\E/);
471    
472            if ($eval_code) {
473                    my $eval = $self->fill_in($rec,$eval_code,$i);
474                    return if (! eval $eval);
475            }
476    
477            return $out;
478    }
479    
480    =head2 parse_to_arr
481    
482    Similar to C<parse>, but returns array of all repeatable fields
483    
484     my @arr = $webpac->parse_to_arr($rec,'v250^a');
485    
486    =cut
487    
488    sub parse_to_arr {
489            my $self = shift;
490    
491            my ($rec, $format_utf8) = @_;
492    
493            confess("need HASH as first argument!") if ($rec !~ /HASH/o);
494            return if (! $format_utf8);
495    
496            my $i = 0;
497            my @arr;
498    
499            while (my $v = $self->parse($rec,$format_utf8,$i++)) {
500                    push @arr, $v;
501            }
502    
503            return @arr;
504    }
505    
506    =head2 data_structure
507    
508    Create in-memory data structure which represents layout from C<import_xml>.
509    It is used later to produce output.
510    
511     my @ds = $webpac->data_structure($rec);
512    
513    =cut
514    
515    # private method _sort_by_order
516    # sort subrouting using order="" attribute
517    sub _sort_by_order {
518            my $self = shift;
519    
520            my $va = $self->{'import_xml'}->{'indexer'}->{$a}->{'order'} ||
521                    $self->{'import_xml'}->{'indexer'}->{$a};
522            my $vb = $self->{'import_xml'}->{'indexer'}->{$b}->{'order'} ||
523                    $self->{'import_xml'}->{'indexer'}->{$b};
524    
525            return $va <=> $vb;
526    }
527    
528    sub data_structure {
529            my $self = shift;
530    
531            my $rec = shift;
532            confess("need HASH as first argument!") if ($rec !~ /HASH/o);
533    
534            my @sorted_tags;
535            if ($self->{tags_by_order}) {
536                    @sorted_tags = @{$self->{tags_by_order}};
537            } else {
538                    @sorted_tags = sort { $self->_sort_by_order } keys %{$self->{'import_xml'}->{'indexer'}};
539                    $self->{tags_by_order} = \@sorted_tags;
540            }
541    
542            my @ds;
543    
544            foreach my $field (@sorted_tags) {
545    
546                    my $row;
547    
548    #print "field $field [",$self->{'tag'},"] = ",Dumper($self->{'import_xml'}->{'indexer'}->{$field}->{$self->{'tag'}});
549    
550                    foreach my $tag (@{$self->{'import_xml'}->{'indexer'}->{$field}->{$self->{'tag'}}}) {
551                            my @v = $self->parse_to_arr($rec,$tag->{'content'});
552    
553                            next if (! @v);
554    
555                            # does tag have type?
556                            if ($tag->{'type'}) {
557                                    push @{$row->{$tag->{'type'}}}, @v;
558                            } else {
559                                    push @{$row->{'display'}}, @v;
560                                    push @{$row->{'swish'}}, @v;
561                            }
562                    }
563    
564                    if ($row) {
565                            $row->{'tag'} = $field;
566                            push @ds, $row;
567                    }
568    
569            }
570    
571          print Dumper(@out);          print "data_structure => ",Dumper(\@ds);
572    
573  }  }
574    

Legend:
Removed from v.357  
changed lines
  Added in v.368

  ViewVC Help
Powered by ViewVC 1.1.26