trunk/search/nn-swish.cgi

#!/usr/bin/perl -w

use strict;
use CGI qw/:standard -no_xhtml/;
use CGI::Carp qw(fatalsToBrowser);
use SWISH;
use Unicode::String qw(utf8 utf16);
use Lingua::Spelling::Alternative;
require Unicode::Map8;
use GDBM_File;
use lib '/data/swish/html';
use FormatResult;

my $dir='/home/dpavlin/nn';
my $prog='/usr/bin/swish-e';
my $url='http://www.nn.hr/clanci/sluzbeno/';

my $hits=0;
my $max_hits=100;

my %labels = (100=>' 100', 200=>' 200', 500=>' 500', 0=>'neogranièeno');
my %index = ('title'=>'naslovu', 'text'=>'tekstu');

my %god_lables;

my %brzakona;
tie %brzakona, 'GDBM_File', "$dir/swish/brzakona.gdbm", &GDBM_READER, 0640 || die "tie: $!";
foreach (sort keys %brzakona) {
        $god_lables{$_} = sprintf("%-8s (%d zakona)",$_,$brzakona{$_});
}
untie %brzakona;

my %file2title;
tie %file2title, 'GDBM_File', "$dir/swish/file2title.gdbm", &GDBM_READER, 0640 || die "tie: $!";

my $hr = new Lingua::Spelling::Alternative( DEBUG => 0 );
$hr->load_findaffix("$dir/prvih_50.txt");

print header(-charset=>'iso-8859-2'),start_html(-title=>'NN - Narodne Novine pretrazivanje',-lang=>'hr'),start_form(-name=>'search_form');
print "Potra¾i ",popup_menu(-name=>'max_hits',-values=>[ sort keys %labels ],-labels=>\%labels,-default=>$max_hits)," zakona sa rijeèima: ",textfield('search');
print " u ",popup_menu(-name=>'index',-values=>[ sort keys %index ],-labels=>\%index,-default=>'title')," zakona ";
print submit(-value=>'prika¾i');
print br,checkbox(-name=>'god_limit', -checked=>0, -label=>"ogranièi pretra¾ivanje samo na godinu "),popup_menu(-name=>'god',-values=>[sort keys %god_lables],-labels=>\%god_lables,-onChange=>'this.form.god_limit.checked=true');
print end_form,hr;

if (param('search')) {

        my $s;
        # re-write query from +/- to and/and not

        my @hl_words;

        foreach (split(/\s+/,param('search'))) {
                if (m/^([+-])(\S+)/) {
                        $s.= ($s) ? "and " : "";
                        $s.="not " if ($1 eq "-");
                        my @alt = $hr->alternatives($2);
                        $s.="(".join("* or ",@alt).") ";
                        push @hl_words, \@alt if ($1 ne "-");
                } else {
                        my @alt = $hr->alternatives($_);
                        $s .= "(".join("* or ",@alt).") ";
                        push @hl_words, \@alt;
                }
        }
        $s=~tr/ðèæÐÈÆ/¹ð¾èæ©Ð®ÈÆ/;  # 1250 -> iso8859-2
        $s=~tr/¹©ðÐèÈæÆ¾®/sSdDcCcCzZ/;

        my $l2_map = Unicode::Map8->new("ISO-8859-2") || die;
        my $us = Unicode::String->new();

        my $sw_q;
        my $sh;

        if (defined(param('max_hits'))) {
                $max_hits = param('max_hits');
        }

        if (param('index') eq 'title') {

                $sh = SWISH->connect('Fork',
                        prog     => $prog,
                        indexes  => "$dir/swish/nn.index",
                        properties  => [qw/god br nr/],
                        results  => sub {
                                my ($sh,$hit) = @_;

                                $us->utf8($hit->swishtitle);

                                my $naslov = $l2_map->to8($us->utf16);
                                $naslov = FormatResult::highlite_words(\$naslov, \@hl_words);
                                print "<tt><a href=\"$url",$hit->swishdocpath,"\">NN",$hit->god,"/",$hit->br,"</a> ",$hit->nr," </tt> $naslov [",$hit->swishrank,"]<br>\n";
        #                       print $_[1]->as_string,"<br>\n";
        #                       my @fields = $hit->field_names;
        #                       print "Field '$_' = '", $hit->$_, "'<br>\n" for sort @fields;
                        },
                        maxhits => $max_hits,
                );

                die $SWISH::errstr unless $sh;

                $sw_q = "naslov_czs=($s)";
                if (param('god_limit')) {
                        $sw_q .= " and god=".int(param('god')) if (param('god'));
                }

        } else {
                # search in full text

                $sh = SWISH->connect('Fork',
                        prog     => $prog,
                        indexes  => "$dir/swish/sluzbeno.index",
                        results  => sub {
                                my ($sh,$hit) = @_;

                                my $path = $hit->swishdocpath;
                                if ($file2title{$path}) {
                                        my ($god,$br,$nr,undef,$naslov) = split(/ /,$file2title{$path},5);
                                        $naslov = FormatResult::highlite_words(\$naslov, \@hl_words);
                                        print "<tt><a href=\"$url",$path,"\">NN$god/$br</a> $nr</tt> $naslov [",$hit->swishrank,"]<br>\n";

                                } else {
                                        print "<!-- error! can't find $path -->\n";
                                }

                        },
                        maxhits => $max_hits,
                );

                die $SWISH::errstr unless $sh;

                $sw_q = $s;
                if (param('god_limit')) {
                        $sw_q .= " and swishdocpath=".int(param('god')) if (param('god'));
                }

        }

        print "<!-- swish query: $sw_q -->";

        $hits = $sh->query($sw_q);

        if ($hits > 0) {
                print p,hr,"Prikazujem $hits zakona";
                print " iz godine ",param('god') if (param('god_limit'));
                print " (maks. $max_hits)... " if ($max_hits);
                print " <small>[$s]</small>";
        } else {
                print p,"Nije naðen niti jedan zakon... <small>[$s, ",$sh->errstr,"]</small>";
        }
} else {
        my $dir=$0;
        $dir=~s,(^.*?)/[^/]+$,$1,g;
        open(HELP, "$dir/nn-help.html") || die "can't open '$dir/nn-help.html'";
        while(<HELP>) {
                print;
        }
        close(HELP);
}

untie %file2title;
1	dpavlin	68	#!/usr/bin/perl -w
2
3			use strict;
4			use CGI qw/:standard -no_xhtml/;
5			use CGI::Carp qw(fatalsToBrowser);
6			use SWISH;
7	dpavlin	74	use Unicode::String qw(utf8 utf16);
8	dpavlin	91	use Lingua::Spelling::Alternative;
9	dpavlin	71	require Unicode::Map8;
10	dpavlin	90	use GDBM_File;
11	dpavlin	97	use lib '/data/swish/html';
12			use FormatResult;
13	dpavlin	68
14	dpavlin	91	my $dir='/home/dpavlin/nn';
15	dpavlin	86	my $prog='/usr/bin/swish-e';
16	dpavlin	91	my $url='http://www.nn.hr/clanci/sluzbeno/';
17	dpavlin	68
18			my $hits=0;
19	dpavlin	77	my $max_hits=100;
20	dpavlin	68
21	dpavlin	77	my %labels = (100=>' 100', 200=>' 200', 500=>' 500', 0=>'neogranièeno');
22	dpavlin	91	my %index = ('title'=>'naslovu', 'text'=>'tekstu');
23	dpavlin	77
24	dpavlin	95	my %god_lables;
25	dpavlin	90
26	dpavlin	91	my %brzakona;
27			tie %brzakona, 'GDBM_File', "$dir/swish/brzakona.gdbm", &GDBM_READER, 0640 \|\| die "tie: $!";
28			foreach (sort keys %brzakona) {
29	dpavlin	95	$god_lables{$_} = sprintf("%-8s (%d zakona)",$_,$brzakona{$_});
30	dpavlin	90	}
31	dpavlin	91	untie %brzakona;
32	dpavlin	90
33	dpavlin	91	my %file2title;
34			tie %file2title, 'GDBM_File', "$dir/swish/file2title.gdbm", &GDBM_READER, 0640 \|\| die "tie: $!";
35
36			my $hr = new Lingua::Spelling::Alternative( DEBUG => 0 );
37			$hr->load_findaffix("$dir/prvih_50.txt");
38
39	dpavlin	97	print header(-charset=>'iso-8859-2'),start_html(-title=>'NN - Narodne Novine pretrazivanje',-lang=>'hr'),start_form(-name=>'search_form');
40	dpavlin	77	print "Potra¾i ",popup_menu(-name=>'max_hits',-values=>[ sort keys %labels ],-labels=>\%labels,-default=>$max_hits)," zakona sa rijeèima: ",textfield('search');
41	dpavlin	91	print " u ",popup_menu(-name=>'index',-values=>[ sort keys %index ],-labels=>\%index,-default=>'title')," zakona ";
42	dpavlin	68	print submit(-value=>'prika¾i');
43	dpavlin	95	print br,checkbox(-name=>'god_limit', -checked=>0, -label=>"ogranièi pretra¾ivanje samo na godinu "),popup_menu(-name=>'god',-values=>[sort keys %god_lables],-labels=>\%god_lables,-onChange=>'this.form.god_limit.checked=true');
44	dpavlin	68	print end_form,hr;
45
46			if (param('search')) {
47
48	dpavlin	76	my $s;
49			# re-write query from +/- to and/and not
50	dpavlin	97
51			my @hl_words;
52
53	dpavlin	76	foreach (split(/\s+/,param('search'))) {
54			if (m/^([+-])(\S+)/) {
55			$s.= ($s) ? "and " : "";
56			$s.="not " if ($1 eq "-");
57	dpavlin	97	my @alt = $hr->alternatives($2);
58			$s.="(".join("* or ",@alt).") ";
59			push @hl_words, \@alt if ($1 ne "-");
60	dpavlin	76	} else {
61	dpavlin	97	my @alt = $hr->alternatives($_);
62			$s .= "(".join("* or ",@alt).") ";
63			push @hl_words, \@alt;
64	dpavlin	76	}
65			}
66	dpavlin	68	$s=~tr/ðèæÐÈÆ/¹ð¾èæ©Ð®ÈÆ/; # 1250 -> iso8859-2
67			$s=~tr/¹©ðÐèÈæÆ¾®/sSdDcCcCzZ/;
68
69	dpavlin	71	my $l2_map = Unicode::Map8->new("ISO-8859-2") \|\| die;
70	dpavlin	74	my $us = Unicode::String->new();
71	dpavlin	71
72	dpavlin	91	my $sw_q;
73			my $sh;
74	dpavlin	70
75	dpavlin	101	if (defined(param('max_hits'))) {
76			$max_hits = param('max_hits');
77			}
78
79	dpavlin	91	if (param('index') eq 'title') {
80	dpavlin	70
81	dpavlin	91	$sh = SWISH->connect('Fork',
82			prog => $prog,
83			indexes => "$dir/swish/nn.index",
84			properties => [qw/god br nr/],
85			results => sub {
86			my ($sh,$hit) = @_;
87	dpavlin	71
88	dpavlin	91	$us->utf8($hit->swishtitle);
89	dpavlin	68
90	dpavlin	97	my $naslov = $l2_map->to8($us->utf16);
91			$naslov = FormatResult::highlite_words(\$naslov, \@hl_words);
92			print "<tt><a href=\"$url",$hit->swishdocpath,"\">NN",$hit->god,"/",$hit->br,"</a> ",$hit->nr," </tt> $naslov [",$hit->swishrank,"]<br>\n";
93	dpavlin	91	# print $_[1]->as_string,"<br>\n";
94			# my @fields = $hit->field_names;
95			# print "Field '$_' = '", $hit->$_, "'<br>\n" for sort @fields;
96			},
97	dpavlin	101	maxhits => $max_hits,
98	dpavlin	91	);
99	dpavlin	68
100	dpavlin	91	die $SWISH::errstr unless $sh;
101
102			$sw_q = "naslov_czs=($s)";
103	dpavlin	95	if (param('god_limit')) {
104			$sw_q .= " and god=".int(param('god')) if (param('god'));
105	dpavlin	91	}
106
107			} else {
108			# search in full text
109
110			$sh = SWISH->connect('Fork',
111			prog => $prog,
112			indexes => "$dir/swish/sluzbeno.index",
113			results => sub {
114			my ($sh,$hit) = @_;
115
116			my $path = $hit->swishdocpath;
117			if ($file2title{$path}) {
118			my ($god,$br,$nr,undef,$naslov) = split(/ /,$file2title{$path},5);
119	dpavlin	97	$naslov = FormatResult::highlite_words(\$naslov, \@hl_words);
120	dpavlin	91	print "<tt><a href=\"$url",$path,"\">NN$god/$br</a> $nr</tt> $naslov [",$hit->swishrank,"]<br>\n";
121
122			} else {
123			print "<!-- error! can't find $path -->\n";
124			}
125
126			},
127	dpavlin	101	maxhits => $max_hits,
128	dpavlin	91	);
129
130			die $SWISH::errstr unless $sh;
131
132			$sw_q = $s;
133	dpavlin	95	if (param('god_limit')) {
134			$sw_q .= " and swishdocpath=".int(param('god')) if (param('god'));
135	dpavlin	91	}
136
137	dpavlin	90	}
138
139			print "<!-- swish query: $sw_q -->";
140
141			$hits = $sh->query($sw_q);
142
143	dpavlin	68	if ($hits > 0) {
144	dpavlin	90	print p,hr,"Prikazujem $hits zakona";
145	dpavlin	95	print " iz godine ",param('god') if (param('god_limit'));
146	dpavlin	101	print " (maks. $max_hits)... " if ($max_hits);
147			print " <small>[$s]</small>";
148	dpavlin	68	} else {
149	dpavlin	101	print p,"Nije naðen niti jedan zakon... <small>[$s, ",$sh->errstr,"]</small>";
150	dpavlin	68	}
151			} else {
152	dpavlin	95	my $dir=$0;
153			$dir=~s,(^.*?)/[^/]+$,$1,g;
154			open(HELP, "$dir/nn-help.html") \|\| die "can't open '$dir/nn-help.html'";
155			while(<HELP>) {
156			print;
157			}
158			close(HELP);
159	dpavlin	68	}
160	dpavlin	91
161			untie %file2title;