webpac/trunk2/all2all.pl

#!/usr/bin/perl -w

=head1 NAME

all2all.pl - basic script for all WebPAC needs

=cut

use strict;
use YAML;
use Carp;
use Getopt::Long;

use lib './lib';
use WebPAC;
use WebPAC::jsFind;
use WebPAC::Index;
use WebPAC::Tree;

# options which can be changed via command line
#
my $code_page = 'ISO-8859-2';
my ($limit_mfn, $start_mfn, $debug, $low_mem);
my $index_path = './out/index';

my $result = GetOptions(
        "code_page=s"   => \$code_page,
        "limit_mfn=i"   => \$limit_mfn,
        "start_mfn=i"   => \$start_mfn,
        "debug!"        => \$debug,
        "low_mem!"      => \$low_mem,
);

# create WebPAC object
#
my $webpac = new WebPAC(
        code_page => $code_page,
        limit_mfn => $limit_mfn,
        start_mfn => $start_mfn,
        debug => $debug,
        low_mem => $low_mem,
) || die;

my $log = $webpac->_get_logger() || die "can't get logger";

$log->debug("creating WebPAC::jsFind object");

my $index = new WebPAC::jsFind(
        index_path => $index_path,
        keys => 10,
) || die;

my $thes;

$|=1;

my $maxmfn = $webpac->open_isis(
        filename => shift @ARGV || '/data/hidra/THS/THS',
        lookup => [
        { 'key' => 'd:v900', 'val' => 'v250^a' },
#       { 'eval' => '"v901^a" eq "Podruèje"', 'key' => 'pa:v561^4:v562^4:v461^1', 'val' => 'v900' },
#       { 'eval '=> '"v901^a" eq "Mikrotezaurus"', 'key' => 'a:v561^4:v562^4:v461^1', 'val' => 'v900' },
#       { 'eval' => '"v901^a" eq "Deskriptor"', 'key' => 'a:v561^4:v562^4:v461^1', 'val' => 'v900' },
        { 'key' => 'a:v561^4:v562^4:v461^1', 'val' => 'v900' },
        { 'key' => '900_mfn:v900', 'val' => 'v000' },
        # tree structure
        { 'eval' => 'length("v251") == 2', 'key' => 'root:v251', 'val' => 'v900' },
        { 'eval' => '"v251"', 'key' => 'code:v900', 'val' => 'v561^4:v251' },
        { 'eval' => '"v561^4" && "v562^4"', 'key' => 'code:v900', 'val' => 'v561^4:v562^4' },
        ],
);

$log->debug("isis file ",$webpac->{'isis_filename'}," opened");

$log->info("rows: $maxmfn");

$webpac->open_import_xml(type => 'isis_hidra_ths');

if(1) { # XXX

while (my $rec = $webpac->fetch_rec) {

        my @ds = $webpac->data_structure($rec);

        if (0 && $log->is_debug) {
                $log->debug("rec = ",Dump($rec));
                $log->debug("ds = ",Dump(\@ds));
        }

        next if (! @ds);

        my $filename = $webpac->{'current_filename'} || $log->logdie("no current_filename in webpac object");

        if ($filename) {
                $webpac->output_file(
                        file => $filename,
                        template => 'html.tt',
                        data => \@ds,
                        headline => $webpac->{'headline'},
                );
        } else {
                print $webpac->output(
                        template => 'text.tt',
                        data => \@ds,
                        headline => $webpac->{'headline'},
                );
        }

        my $headline = $webpac->{'headline'};

        my $f = $filename;
        $f =~ s!out/!!;

        # save into index
        foreach my $ds (@ds) {
                next if (! $ds->{'swish'});

                $index->insert(
                        index_name => $ds->{'tag'},
                        path => $f,
                        headline => $headline,
                        words => join(" ",@{$ds->{'swish'}})
                );
        }

        # save into sorted index (thesaurus)
        foreach my $ds (@ds) {
                next if (! $ds->{'index'});

                $thes->{$ds->{'tag'}} ||= new WebPAC::Index;

                foreach my $h (@{$ds->{'index'}}) {
                        $thes->{$ds->{'tag'}}->insert(
                                headline => $h,
                                mfn => $webpac->mfn,
                        );
                }
        }

#       print Dump(\@ds);

}

foreach my $t (keys %{$thes}) {

        my @e = $thes->{$t}->elements;
        if (! @e) {
                $log->logwarn("no elements in sorted index $t?");
                next;
        }

        my $file = "./out/bfilter/$t.txt";
        $log->info("saving sorted index $t to '$file' [".scalar(@e)." elements]");

        $webpac->output_file(
                file => $file,
                template => 'index.tt',
                data => \@e,
                index_name => $t,
        );
}

if (0 && $log->is_debug) {
        $log->debug("lookup hash: ",Dump($webpac->{'lookup'}));
        $log->debug("data hash: ",Dump($webpac->{'data'}));
        foreach my $t (keys %{$thes}) {
                $log->debug("thesaurus $t hash: ",Dump($thes->{$t}));
        }
}

} # XXX if(0)

$log->debug("lookup hash: ",Dump($webpac->{'lookup'}));

$log->info("creating tree");

#
# define tree structure
#

my $l = $webpac->{'lookup'} || $log->logconfess("can't find lookup");

my @tree = ({
        # level 0
        code_arr        => sub { sort keys %{$l} },
        filter_code     => sub {
                                        my $t = shift;
                                        return $t if ($t =~ s/root://);
                                },
        lookup_v900     => sub { shift @{$l->{"root:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[1]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[1]}} },
        have_children   => sub { return $l->{"a:".$_[0]."::"} },
        have_children_at_level => sub {
                                my $l2 = shift;
                                my $l1 = substr($l2,0,1);
                                return(9, "a:$l1:$l2:" ) if ($l->{"a:$l1:$l2"});
                        },
        style           => 'display: none',
        },{
        # 1
        code_arr        => sub { @{$_[0]} },
        filter_code     => sub { shift },       # nop
        lookup_v900     => sub { shift @{$l->{"code:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[0]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[0]}} },
        have_children   => sub { return $l->{"a:".$_[1].":"} },
        style           => 'display: none',
        },{
        # 2
        code_arr        => sub { @{$_[0]} },
        filter_code     => sub { shift },
        lookup_v900     => sub { shift @{$l->{"code:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[0]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[0]}} },
        have_children   => sub { return $l->{"a:".$_[1].":".$_[0]} },
        #style          => 'display: none',
        },{
        # 3 u¾i pojam
        code_arr        => sub { @{$_[0]} },
        filter_code     => sub { shift },
        lookup_v900     => sub { shift @{$l->{"code:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[0]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[0]}} },
        have_children   => sub { return $l->{"a:".$_[1].":".$_[0]} },
        },{
        # 4
        code_arr        => sub { @{$_[0]} },
        filter_code     => sub { shift },
        lookup_v900     => sub { shift @{$l->{"code:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[0]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[0]}} },
        have_children   => sub { return $l->{"a:".$_[1].":".$_[0]} },
        },{
        # 5
        code_arr        => sub { @{$_[0]} },
        filter_code     => sub { shift },
        lookup_v900     => sub { shift @{$l->{"code:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[0]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[0]}} },
        have_children   => sub { return $l->{"a:".$_[1].":".$_[0]} },
        },{
        # 6
        code_arr        => sub { @{$_[0]} },
        filter_code     => sub { shift },
        lookup_v900     => sub { shift @{$l->{"code:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[0]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[0]}} },
        have_children   => sub { return $l->{"a:".$_[1].":".$_[0]} },
        },{
        # 7
        code_arr        => sub { @{$_[0]} },
        filter_code     => sub { shift },
        lookup_v900     => sub { shift @{$l->{"code:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[0]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[0]}} },
        have_children   => sub { return $l->{"a:".$_[1].":".$_[0]} },
        },{
        # 8
        code_arr        => sub { @{$_[0]} },
        filter_code     => sub { shift },
        lookup_v900     => sub { shift @{$l->{"code:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[0]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[0]}} },
#       have_children   => sub { return $l->{"a:".$_[1].":".$_[0]} },
        have_children   => sub { 0 },
        },{
        # 9 - level which is never reached except explicitly
        code_arr        => sub { @{$l->{$_[0]}} },
        filter_code     => sub { shift },
        lookup_v900     => sub { shift @{$l->{"code:".$_[0]}} },
        lookup_term     => sub { shift @{$l->{"d:".$_[0]}} },
        lookup_mfn      => sub { shift @{$l->{"900_mfn:".$_[0]}} },
        have_children   => sub { 0 },
        have_children_at_level => sub { defined($l->{"a:".$_[1].":".$_[0]}) && return (9,"a:".$_[1].":".$_[0]) },
        },{
});

my $tree = new WebPAC::Tree(
        dir => './out',
        html => 'browse.html',
        template => './output_template/tree.tt',
        js => 'tree-ids.js',
        tree => \@tree,
);

$log->info("closing index");
$index->close;

$log->info("elapsed time: ",$webpac->fmt_time(time() - $webpac->{'start_t'}));
1	dpavlin	1	#!/usr/bin/perl -w
2
3	dpavlin	348	=head1 NAME
4
5	dpavlin	354	all2all.pl - basic script for all WebPAC needs
6	dpavlin	348
7			=cut
8
9	dpavlin	1	use strict;
10	dpavlin	454	use YAML;
11	dpavlin	352	use Carp;
12	dpavlin	437	use Getopt::Long;
13	dpavlin	1
14	dpavlin	352	use lib './lib';
15	dpavlin	354	use WebPAC;
16	dpavlin	390	use WebPAC::jsFind;
17	dpavlin	410	use WebPAC::Index;
18	dpavlin	441	use WebPAC::Tree;
19	dpavlin	348
20	dpavlin	437	# options which can be changed via command line
21			#
22			my $code_page = 'ISO-8859-2';
23			my ($limit_mfn, $start_mfn, $debug, $low_mem);
24			my $index_path = './out/index';
25
26			my $result = GetOptions(
27			"code_page=s" => \$code_page,
28			"limit_mfn=i" => \$limit_mfn,
29			"start_mfn=i" => \$start_mfn,
30			"debug!" => \$debug,
31			"low_mem!" => \$low_mem,
32			);
33
34			# create WebPAC object
35			#
36	dpavlin	354	my $webpac = new WebPAC(
37	dpavlin	437	code_page => $code_page,
38			limit_mfn => $limit_mfn,
39			start_mfn => $start_mfn,
40			debug => $debug,
41			low_mem => $low_mem,
42	dpavlin	352	) \|\| die;
43
44	dpavlin	421	my $log = $webpac->_get_logger() \|\| die "can't get logger";
45	dpavlin	373
46	dpavlin	421	$log->debug("creating WebPAC::jsFind object");
47
48	dpavlin	390	my $index = new WebPAC::jsFind(
49	dpavlin	437	index_path => $index_path,
50	dpavlin	413	keys => 10,
51	dpavlin	390	) \|\| die;
52
53	dpavlin	410	my $thes;
54
55	dpavlin	10	$\|=1;
56	dpavlin	9
57	dpavlin	352	my $maxmfn = $webpac->open_isis(
58	dpavlin	357	filename => shift @ARGV \|\| '/data/hidra/THS/THS',
59	dpavlin	352	lookup => [
60	dpavlin	348	{ 'key' => 'd:v900', 'val' => 'v250^a' },
61	dpavlin	352	# { 'eval' => '"v901^a" eq "Podruèje"', 'key' => 'pa:v561^4:v562^4:v461^1', 'val' => 'v900' },
62			# { 'eval '=> '"v901^a" eq "Mikrotezaurus"', 'key' => 'a:v561^4:v562^4:v461^1', 'val' => 'v900' },
63			# { 'eval' => '"v901^a" eq "Deskriptor"', 'key' => 'a:v561^4:v562^4:v461^1', 'val' => 'v900' },
64			{ 'key' => 'a:v561^4:v562^4:v461^1', 'val' => 'v900' },
65	dpavlin	374	{ 'key' => '900_mfn:v900', 'val' => 'v000' },
66	dpavlin	419	# tree structure
67			{ 'eval' => 'length("v251") == 2', 'key' => 'root:v251', 'val' => 'v900' },
68			{ 'eval' => '"v251"', 'key' => 'code:v900', 'val' => 'v561^4:v251' },
69	dpavlin	460	{ 'eval' => '"v561^4" && "v562^4"', 'key' => 'code:v900', 'val' => 'v561^4:v562^4' },
70	dpavlin	352	],
71	dpavlin	54	);
72	dpavlin	3
73	dpavlin	421	$log->debug("isis file ",$webpac->{'isis_filename'}," opened");
74
75	dpavlin	372	$log->info("rows: $maxmfn");
76	dpavlin	170
77	dpavlin	363	$webpac->open_import_xml(type => 'isis_hidra_ths');
78
79	dpavlin	425	if(1) { # XXX
80	dpavlin	424
81	dpavlin	362	while (my $rec = $webpac->fetch_rec) {
82	dpavlin	358
83	dpavlin	370	my @ds = $webpac->data_structure($rec);
84	dpavlin	366
85	dpavlin	411	if (0 && $log->is_debug) {
86	dpavlin	454	$log->debug("rec = ",Dump($rec));
87			$log->debug("ds = ",Dump(\@ds));
88	dpavlin	372	}
89
90	dpavlin	374	next if (! @ds);
91	dpavlin	366
92	dpavlin	421	my $filename = $webpac->{'current_filename'} \|\| $log->logdie("no current_filename in webpac object");
93	dpavlin	374
94			if ($filename) {
95	dpavlin	411	$webpac->output_file(
96			file => $filename,
97	dpavlin	374	template => 'html.tt',
98			data => \@ds,
99	dpavlin	398	headline => $webpac->{'headline'},
100	dpavlin	374	);
101			} else {
102			print $webpac->output(
103			template => 'text.tt',
104			data => \@ds,
105	dpavlin	398	headline => $webpac->{'headline'},
106	dpavlin	374	);
107			}
108
109	dpavlin	398	my $headline = $webpac->{'headline'};
110	dpavlin	390
111	dpavlin	392	my $f = $filename;
112			$f =~ s!out/!!;
113
114	dpavlin	390	# save into index
115			foreach my $ds (@ds) {
116			next if (! $ds->{'swish'});
117
118			$index->insert(
119			index_name => $ds->{'tag'},
120	dpavlin	392	path => $f,
121	dpavlin	390	headline => $headline,
122			words => join(" ",@{$ds->{'swish'}})
123			);
124			}
125
126	dpavlin	410	# save into sorted index (thesaurus)
127			foreach my $ds (@ds) {
128			next if (! $ds->{'index'});
129
130			$thes->{$ds->{'tag'}} \|\|= new WebPAC::Index;
131
132	dpavlin	415	foreach my $h (@{$ds->{'index'}}) {
133			$thes->{$ds->{'tag'}}->insert(
134			headline => $h,
135	dpavlin	448	mfn => $webpac->mfn,
136	dpavlin	415	);
137			}
138	dpavlin	410	}
139
140	dpavlin	454	# print Dump(\@ds);
141	dpavlin	390
142	dpavlin	3	}
143
144	dpavlin	411	foreach my $t (keys %{$thes}) {
145
146			my @e = $thes->{$t}->elements;
147			if (! @e) {
148			$log->logwarn("no elements in sorted index $t?");
149			next;
150			}
151
152	dpavlin	415	my $file = "./out/bfilter/$t.txt";
153			$log->info("saving sorted index $t to '$file' [".scalar(@e)." elements]");
154	dpavlin	411
155			$webpac->output_file(
156	dpavlin	415	file => $file,
157	dpavlin	411	template => 'index.tt',
158			data => \@e,
159			index_name => $t,
160			);
161			}
162
163			if (0 && $log->is_debug) {
164	dpavlin	454	$log->debug("lookup hash: ",Dump($webpac->{'lookup'}));
165			$log->debug("data hash: ",Dump($webpac->{'data'}));
166	dpavlin	410	foreach my $t (keys %{$thes}) {
167	dpavlin	454	$log->debug("thesaurus $t hash: ",Dump($thes->{$t}));
168	dpavlin	410	}
169	dpavlin	372	}
170	dpavlin	390
171	dpavlin	424	} # XXX if(0)
172
173	dpavlin	454	$log->debug("lookup hash: ",Dump($webpac->{'lookup'}));
174	dpavlin	420
175	dpavlin	441	$log->info("creating tree");
176
177	dpavlin	419	#
178	dpavlin	441	# define tree structure
179	dpavlin	419	#
180
181	dpavlin	424	my $l = $webpac->{'lookup'} \|\| $log->logconfess("can't find lookup");
182	dpavlin	419
183	dpavlin	424	my @tree = ({
184			# level 0
185			code_arr => sub { sort keys %{$l} },
186			filter_code => sub {
187			my $t = shift;
188			return $t if ($t =~ s/root://);
189			},
190			lookup_v900 => sub { shift @{$l->{"root:".$_[0]}} },
191			lookup_term => sub { shift @{$l->{"d:".$_[1]}} },
192			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[1]}} },
193	dpavlin	454	have_children => sub { return $l->{"a:".$_[0]."::"} },
194			have_children_at_level => sub {
195			my $l2 = shift;
196			my $l1 = substr($l2,0,1);
197			return(9, "a:$l1:$l2:" ) if ($l->{"a:$l1:$l2"});
198			},
199	dpavlin	424	style => 'display: none',
200			},{
201			# 1
202	dpavlin	454	code_arr => sub { @{$_[0]} },
203	dpavlin	424	filter_code => sub { shift }, # nop
204			lookup_v900 => sub { shift @{$l->{"code:".$_[0]}} },
205			lookup_term => sub { shift @{$l->{"d:".$_[0]}} },
206			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[0]}} },
207	dpavlin	454	have_children => sub { return $l->{"a:".$_[1].":"} },
208	dpavlin	424	style => 'display: none',
209			},{
210			# 2
211	dpavlin	454	code_arr => sub { @{$_[0]} },
212	dpavlin	424	filter_code => sub { shift },
213	dpavlin	460	lookup_v900 => sub { shift @{$l->{"code:".$_[0]}} },
214	dpavlin	424	lookup_term => sub { shift @{$l->{"d:".$_[0]}} },
215			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[0]}} },
216	dpavlin	460	have_children => sub { return $l->{"a:".$_[1].":".$_[0]} },
217	dpavlin	454	#style => 'display: none',
218	dpavlin	425	},{
219			# 3 u¾i pojam
220	dpavlin	454	code_arr => sub { @{$_[0]} },
221	dpavlin	425	filter_code => sub { shift },
222	dpavlin	460	lookup_v900 => sub { shift @{$l->{"code:".$_[0]}} },
223	dpavlin	426	lookup_term => sub { shift @{$l->{"d:".$_[0]}} },
224			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[0]}} },
225	dpavlin	460	have_children => sub { return $l->{"a:".$_[1].":".$_[0]} },
226	dpavlin	426	},{
227			# 4
228	dpavlin	454	code_arr => sub { @{$_[0]} },
229	dpavlin	426	filter_code => sub { shift },
230	dpavlin	460	lookup_v900 => sub { shift @{$l->{"code:".$_[0]}} },
231	dpavlin	426	lookup_term => sub { shift @{$l->{"d:".$_[0]}} },
232			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[0]}} },
233	dpavlin	460	have_children => sub { return $l->{"a:".$_[1].":".$_[0]} },
234	dpavlin	427	},{
235			# 5
236	dpavlin	454	code_arr => sub { @{$_[0]} },
237	dpavlin	427	filter_code => sub { shift },
238	dpavlin	460	lookup_v900 => sub { shift @{$l->{"code:".$_[0]}} },
239	dpavlin	427	lookup_term => sub { shift @{$l->{"d:".$_[0]}} },
240			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[0]}} },
241	dpavlin	460	have_children => sub { return $l->{"a:".$_[1].":".$_[0]} },
242	dpavlin	427	},{
243			# 6
244	dpavlin	454	code_arr => sub { @{$_[0]} },
245	dpavlin	427	filter_code => sub { shift },
246	dpavlin	460	lookup_v900 => sub { shift @{$l->{"code:".$_[0]}} },
247	dpavlin	427	lookup_term => sub { shift @{$l->{"d:".$_[0]}} },
248			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[0]}} },
249	dpavlin	460	have_children => sub { return $l->{"a:".$_[1].":".$_[0]} },
250	dpavlin	427	},{
251			# 7
252	dpavlin	454	code_arr => sub { @{$_[0]} },
253	dpavlin	427	filter_code => sub { shift },
254	dpavlin	460	lookup_v900 => sub { shift @{$l->{"code:".$_[0]}} },
255	dpavlin	427	lookup_term => sub { shift @{$l->{"d:".$_[0]}} },
256			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[0]}} },
257	dpavlin	460	have_children => sub { return $l->{"a:".$_[1].":".$_[0]} },
258	dpavlin	427	},{
259			# 8
260	dpavlin	454	code_arr => sub { @{$_[0]} },
261	dpavlin	427	filter_code => sub { shift },
262	dpavlin	460	lookup_v900 => sub { shift @{$l->{"code:".$_[0]}} },
263	dpavlin	427	lookup_term => sub { shift @{$l->{"d:".$_[0]}} },
264			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[0]}} },
265	dpavlin	460	# have_children => sub { return $l->{"a:".$_[1].":".$_[0]} },
266	dpavlin	424	have_children => sub { 0 },
267	dpavlin	454	},{
268			# 9 - level which is never reached except explicitly
269			code_arr => sub { @{$l->{$_[0]}} },
270			filter_code => sub { shift },
271	dpavlin	460	lookup_v900 => sub { shift @{$l->{"code:".$_[0]}} },
272	dpavlin	454	lookup_term => sub { shift @{$l->{"d:".$_[0]}} },
273			lookup_mfn => sub { shift @{$l->{"900_mfn:".$_[0]}} },
274			have_children => sub { 0 },
275	dpavlin	460	have_children_at_level => sub { defined($l->{"a:".$_[1].":".$_[0]}) && return (9,"a:".$_[1].":".$_[0]) },
276	dpavlin	454	},{
277	dpavlin	424	});
278	dpavlin	419
279	dpavlin	441	my $tree = new WebPAC::Tree(
280			dir => './out',
281			html => 'browse.html',
282			template => './output_template/tree.tt',
283			js => 'tree-ids.js',
284			tree => \@tree,
285			);
286	dpavlin	428
287			$log->info("closing index");
288	dpavlin	390	$index->close;
289
290	dpavlin	422	$log->info("elapsed time: ",$webpac->fmt_time(time() - $webpac->{'start_t'}));