--- trunk2/all2all.pl 2004/06/13 20:39:44 349 +++ trunk2/all2all.pl 2004/09/11 08:36:38 422 @@ -2,125 +2,232 @@ =head1 NAME -webpac.pl - basic script for all WebPac needs +all2all.pl - basic script for all WebPAC needs =cut use strict; -use OpenIsis; -use Text::Iconv; use Data::Dumper; +use Carp; -my $cp = Text::Iconv->new("852", "iso-8859-2"); +use lib './lib'; +use WebPAC; +use WebPAC::jsFind; +use WebPAC::Index; + +my $webpac = new WebPAC( + code_page => 'ISO-8859-2', +# limit_mfn => 100, +# debug => 1, + low_mem => 1, +) || die; + +my $log = $webpac->_get_logger() || die "can't get logger"; + +$log->debug("creating WebPAC::jsFind object"); + +my $index = new WebPAC::jsFind( + index_path => './out/index', + keys => 10, +) || die; + +my $thes; $|=1; -my $db = OpenIsis::open( shift @ARGV || '/data/hidra/THS-500/THS' ); -my $maxmfn = OpenIsis::maxRowid( $db ) || 1; +my $maxmfn = $webpac->open_isis( + filename => shift @ARGV || '/data/hidra/THS/THS', + lookup => [ + { 'key' => 'd:v900', 'val' => 'v250^a' }, +# { 'eval' => '"v901^a" eq "Područje"', 'key' => 'pa:v561^4:v562^4:v461^1', 'val' => 'v900' }, +# { 'eval '=> '"v901^a" eq "Mikrotezaurus"', 'key' => 'a:v561^4:v562^4:v461^1', 'val' => 'v900' }, +# { 'eval' => '"v901^a" eq "Deskriptor"', 'key' => 'a:v561^4:v562^4:v461^1', 'val' => 'v900' }, + { 'key' => 'a:v561^4:v562^4:v461^1', 'val' => 'v900' }, + { 'key' => '900_mfn:v900', 'val' => 'v000' }, + # tree structure + { 'eval' => 'length("v251") == 2', 'key' => 'root:v251', 'val' => 'v900' }, + { 'eval' => '"v251"', 'key' => 'code:v900', 'val' => 'v561^4:v251' }, + ], +); -print "rows: $maxmfn\n\n"; +$log->debug("isis file ",$webpac->{'isis_filename'}," opened"); -my $data; +$log->info("rows: $maxmfn"); -my @lookup = ( - { 'key' => 'd:v900', 'val' => 'v250^a' }, - { 'eval' => '"v901^a" eq "Područje"', 'key' => 'pa:v561^4:v562^4:v461^1', 'val' => 'v900' }, - { 'eval '=> '"v901^a" eq "Mikrotezaurus"', 'key' => 'ma:v561^4:v562^4:v461^1', 'val' => 'v900' }, - { 'eval' => '"v901^a" eq "Deskriptor"', 'key' => 'da:v561^4:v562^4:v461^1', 'val' => 'v900' }, -); +$webpac->open_import_xml(type => 'isis_hidra_ths'); -sub fill_in { - my $tmp = shift || return; - my $data = shift || return; - my $i = shift || 0; - - my $found = 0; - - sub get_sf { - my ($found,$data,$f,$sf,$i) = @_; - if ($$data->{$f} && $$data->{$f}->[$i]->{$sf}) { - $$found++; - return $$data->{$f}->[$i]->{$sf}; - } else { - return ''; - } - } +$log->debug("data: ",Dumper($webpac->{'data'})); - sub get_nosf { - my ($found,$data,$f,$i) = @_; - if ($$data->{$f} && $$data->{$f}->[$i]) { - $$found++; - return $$data->{$f}->[$i]; - } else { - return ''; - } +while (my $rec = $webpac->fetch_rec) { + + my @ds = $webpac->data_structure($rec); + + if (0 && $log->is_debug) { + $log->debug("rec = ",Dumper($rec)); + $log->debug("ds = ",Dumper(\@ds)); } - $tmp =~ s/v(\d+)\^(\w)/get_sf(\$found,\$data,$1,$2,$i)/ges; - $tmp =~ s/v(\d+)/get_nosf(\$found,\$data,$1,$i)/ges; + next if (! @ds); + + my $filename = $webpac->{'current_filename'} || $log->logdie("no current_filename in webpac object"); - if ($found) { - return $tmp; + if ($filename) { + $webpac->output_file( + file => $filename, + template => 'html.tt', + data => \@ds, + headline => $webpac->{'headline'}, + ); } else { - return; + print $webpac->output( + template => 'text.tt', + data => \@ds, + headline => $webpac->{'headline'}, + ); } -} -print "reading database...\n"; + my $headline = $webpac->{'headline'}; -for (my $mfn = 1; $mfn <= $maxmfn; $mfn++) { + my $f = $filename; + $f =~ s!out/!!; - my $row = OpenIsis::read( $db, $mfn ); - foreach my $k (keys %{$row}) { - if ($k ne "mfn") { - foreach my $l (@{$row->{$k}}) { - $l = $cp->convert($l); - # has subfields? - my $val; - if ($l =~ m/\^/) { - foreach my $t (split(/\^/,$l)) { - next if (! $t); - $val->{substr($t,0,1)} = substr($t,1); - } - } else { - $val = $l; - } + # save into index + foreach my $ds (@ds) { + next if (! $ds->{'swish'}); + + $index->insert( + index_name => $ds->{'tag'}, + path => $f, + headline => $headline, + words => join(" ",@{$ds->{'swish'}}) + ); + } - push @{$data->{$mfn}->{$k}}, $val; - } + # save into sorted index (thesaurus) + foreach my $ds (@ds) { + next if (! $ds->{'index'}); + + $thes->{$ds->{'tag'}} ||= new WebPAC::Index; + + foreach my $h (@{$ds->{'index'}}) { + $thes->{$ds->{'tag'}}->insert( + path => $f, + headline => $h, + ); } + } + +# print Dumper(\@ds); + +} + +foreach my $t (keys %{$thes}) { + + my @e = $thes->{$t}->elements; + if (! @e) { + $log->logwarn("no elements in sorted index $t?"); + next; + } + + my $file = "./out/bfilter/$t.txt"; + $log->info("saving sorted index $t to '$file' [".scalar(@e)." elements]"); + + $webpac->output_file( + file => $file, + template => 'index.tt', + data => \@e, + index_name => $t, + ); +} +if (0 && $log->is_debug) { + $log->debug("lookup hash: ",Dumper($webpac->{'lookup'})); + $log->debug("data hash: ",Dumper($webpac->{'data'})); + foreach my $t (keys %{$thes}) { + $log->debug("thesaurus $t hash: ",Dumper($thes->{$t})); } +} + +$log->info("lookup hash: ",Dumper($webpac->{'lookup'})); + +# +# tree dump implementation which shouldn't be here :-) +# + +my $file = 'out/browse.html'; +open(HTML, "> $file") || $log->logdie("can't open '$file'"); + +print HTML qq{ + + + Browse + + + + + + +}; + +$index->close; -print Dumper($data); +$log->info("elapsed time: ",$webpac->fmt_time(time() - $webpac->{'start_t'}));