--- trunk2/all2all.pl 2004/10/19 17:43:52 530
+++ trunk2/all2all.pl 2004/10/31 18:43:33 566
@@ -11,6 +11,7 @@
use YAML;
use Carp;
use Getopt::Long;
+use Text::Unaccent 1.02;
use lib './lib';
use WebPAC;
@@ -35,17 +36,61 @@
my $filter = {
'CROVOC' => sub {
my $tmp = shift || return;
- return undef unless ($tmp =~ s/\s*CROVOC.*$/ */);
+ $tmp =~ s/\s*HIDRA.*//;
+ return undef unless ($tmp =~ s/\s*CROVOC.*$/ #C#/);
+ # remove repeating stars
+ # FIXME this should be fixed at right place, not here!
+ $tmp =~ s/(\s*#C#)+/ #C#/g;
return $tmp;
},
'CROVOC_tree' => sub {
my $tmp = shift || return;
- $tmp =~ s/\s*CROVOC.*$/ */;
+ $tmp =~ s/\s*CROVOC.*$/ /;
$tmp =~ s/\s*EUROVOC.*//;
+ $tmp =~ s/\s*HIDRA.*//;
return $tmp;
},
+ # TT filter
+ 'CROVOC_img' => sub {
+ my $tmp = shift;
+ $tmp =~ s/\s*#C#\s*/ /gis;
+ $tmp =~ s/"img\/crovoc.png"/"..\/img\/crovoc.png"/gis;
+ return $tmp;
+ }
};
+## remove accented characters
+#
+sub unac {
+ my $string = shift || return;
+ $string = unac_string($code_page,$string);
+ $string =~ tr/ðÐ/dD/;
+ $string = unac_entities($string);
+ return $string;
+}
+sub unac_2 {
+ my $string = shift || return;
+ if (length($string) > 2) {
+ my $pr = substr($string,0,2);
+ $string = unac_string($code_page,substr($string,2));
+ $string =~ tr/ðÐ/dD/;
+ $string = $pr . $string;
+ }
+ $string = unac_entities($string);
+ return lc($string);
+}
+
+sub unac_entities {
+ my $ent = shift || return;
+
+ $ent =~ s/&(\w)(acute|cedil|circ|grave|ring|slash|tilde|uml);/$1/gi;
+ $ent =~ s/ð/d/g;
+ $ent =~ s/&E[tT][hH];/D/g;
+ $ent =~ s/&(\w\w)lig;/$1/gi;
+
+ return $ent;
+}
+
# create WebPAC object
#
my $webpac = new WebPAC(
@@ -63,7 +108,7 @@
my $index = new WebPAC::jsFind(
index_path => $index_path,
- keys => 10,
+ keys => 62,
) || die;
my $thes;
@@ -150,7 +195,7 @@
#path => $f,
path => $webpac->mfn,
headline => $h,
- words => $words,
+ words => unac($words),
);
}
@@ -162,8 +207,9 @@
foreach my $h (@{$ds->{'index'}}) {
$thes->{$ds->{'tag'}}->insert(
- headline => $h,
+ sort_by => unac_2($h),
mfn => $webpac->mfn,
+ headline => $h,
);
}
}