--- trunk2/all2all.pl 2004/10/10 11:25:10 501
+++ trunk2/all2all.pl 2004/10/30 23:24:54 561
@@ -11,6 +11,7 @@
use YAML;
use Carp;
use Getopt::Long;
+use Text::Unaccent 1.02;
use lib './lib';
use WebPAC;
@@ -35,17 +36,58 @@
my $filter = {
'CROVOC' => sub {
my $tmp = shift || return;
- return undef unless ($tmp =~ s/\s+CROVOC.*$/ */);
+ return undef unless ($tmp =~ s/\s*CROVOC.*$/ #C#/);
+ # remove repeating stars
+ # FIXME this should be fixed at right place, not here!
+ $tmp =~ s/(\s*#C#)+/ #C#/g;
return $tmp;
},
'CROVOC_tree' => sub {
my $tmp = shift || return;
- $tmp =~ s/\s+CROVOC.*$/ */;
- $tmp =~ s/\s+EUROVOC.*//;
+ $tmp =~ s/\s*CROVOC.*$/ /;
+ $tmp =~ s/\s*EUROVOC.*//;
return $tmp;
},
+ # TT filter
+ 'CROVOC_img' => sub {
+ my $tmp = shift;
+ $tmp =~ s/\s*#C#\s*/ /gis;
+ return $tmp;
+ }
};
+## remove accented characters
+#
+sub unac {
+ my $string = shift || return;
+ $string = unac_string($code_page,$string);
+ $string =~ tr/ðÐ/dD/;
+ $string = unac_entities($string);
+ return $string;
+}
+sub unac_2 {
+ my $string = shift || return;
+ if (length($string) > 2) {
+ my $pr = substr($string,0,2);
+ $string = unac_string($code_page,substr($string,2));
+ $string =~ tr/ðÐ/dD/;
+ $string = $pr . $string;
+ }
+ $string = unac_entities($string);
+ return lc($string);
+}
+
+sub unac_entities {
+ my $ent = shift || return;
+
+ $ent =~ s/&(\w)(acute|cedil|circ|grave|ring|slash|tilde|uml);/$1/gi;
+ $ent =~ s/ð/d/g;
+ $ent =~ s/&E[tT][hH];/D/g;
+ $ent =~ s/&(\w\w)lig;/$1/gi;
+
+ return $ent;
+}
+
# create WebPAC object
#
my $webpac = new WebPAC(
@@ -63,7 +105,7 @@
my $index = new WebPAC::jsFind(
index_path => $index_path,
- keys => 10,
+ keys => 62,
) || die;
my $thes;
@@ -138,12 +180,19 @@
$words =~ s/\W*\s+\W*/ /g;
$words =~ s/\W+$//;
+ # first try to generate headline for this entry from index
+ my $h = $ds->{'index'}->[0];
+ # then, from display
+ $h ||= $ds->{'display'}->[0];
+ # and as last resport, fallback to headline
+ $h ||= $headline;
+
$index->insert(
index_name => $ds->{'tag'},
#path => $f,
path => $webpac->mfn,
- headline => $headline,
- words => $words,
+ headline => $h,
+ words => unac($words),
);
}
@@ -151,12 +200,13 @@
foreach my $ds (@ds) {
next if (! $ds->{'index'});
- $thes->{$ds->{'tag'}} ||= new WebPAC::Index;
+ $thes->{$ds->{'tag'}} ||= new WebPAC::Index( name => $ds->{'tag'} );
foreach my $h (@{$ds->{'index'}}) {
$thes->{$ds->{'tag'}}->insert(
- headline => $h,
+ sort_by => unac_2($h),
mfn => $webpac->mfn,
+ headline => $h,
);
}
}
@@ -194,7 +244,7 @@
} # XXX if(0)
-$log->debug("lookup hash: ",Dump($webpac->{'lookup'}));
+#$log->debug("lookup hash: ",Dump($webpac->{'lookup'}));
$log->info("creating tree");
@@ -303,13 +353,24 @@
});
my $tree = new WebPAC::Tree(
- dir => './out',
- html => 'browse.html',
+ tree => \@tree,
+);
+
+$tree->output(
+ dir => './out',
+ html => 'browse.html',
template => './output_template/tree.tt',
js => 'tree-ids.js',
- tree => \@tree,
);
+$tree->output(
+ dir => './eurovoc',
+ html => 'hijerarhija.html',
+ template => './output_template/hijerarhija.tt',
+ js => 'tree-ids.js',
+);
+
+
$log->info("closing index");
$index->close;