/[webpac]/trunk/all2xml.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /trunk/all2xml.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 10 - (hide annotations)
Thu Jan 16 17:35:54 2003 UTC (21 years, 2 months ago) by dpavlin
File MIME type: text/plain
File size: 5683 byte(s)
bunch of changes: make design more modular, implement index (partial
implementation) and other small and big changes

1 dpavlin 1 #!/usr/bin/perl -w
2    
3     use strict;
4     use OpenIsis;
5     use Getopt::Std;
6     use Data::Dumper;
7     use XML::Simple;
8 dpavlin 5 use Text::Unaccent 1.02; # 1.01 won't compile on my platform,
9 dpavlin 10 use Text::Iconv;
10 dpavlin 1
11    
12 dpavlin 10 $|=1;
13 dpavlin 9
14 dpavlin 10 my $config;
15    
16     $config=XMLin(undef, forcearray => [ 'isis' ], forcecontent => 1);
17    
18     use index_DBI; # there is no other, right now ;-)
19    
20     my $index = new index_DBI(); # open index
21    
22 dpavlin 1 my %opts;
23    
24 dpavlin 7 # usage:
25     # -d directory name
26     # -m multiple directories
27     # -q quiet
28     # -s run swish
29 dpavlin 1
30 dpavlin 7 getopts('d:m:qs', \%opts);
31    
32 dpavlin 3 my $db_dir = $opts{d} || "ps"; # FIX
33 dpavlin 1
34 dpavlin 3 #die "usage: $0 -d [database_dir] -m [database1,database2] " if (! %opts);
35 dpavlin 1
36 dpavlin 3 #print Dumper($config->{indexer});
37     #print "-" x 70,"\n";
38 dpavlin 1
39 dpavlin 10 Text::Iconv->raise_error(1); # Conversion errors raise exceptions
40 dpavlin 3
41 dpavlin 10 my $isis_codepage = Text::Iconv->new($config->{isis_codepage},'UTF8');
42     my $index_codepage = Text::Iconv->new($config->{isis_codepage},$config->{index_codepage});
43     my $cludge_codepage = Text::Iconv->new('UTF8','ISO8859-1');
44    
45 dpavlin 3 sub isis2xml {
46    
47 dpavlin 10 use xmlify;
48    
49 dpavlin 3 my $row = shift @_;
50    
51     my $xml;
52 dpavlin 10 $xml .= xmlify('db_dir',$db_dir); # FIX remove?
53 dpavlin 3
54 dpavlin 10 use parse_format;
55 dpavlin 3
56 dpavlin 10
57 dpavlin 3 foreach my $field (keys %{$config->{indexer}}) {
58    
59 dpavlin 10 my $swish_data = "";
60 dpavlin 3 my $display_data = "";
61     my $index_data = "";
62    
63     foreach my $x (@{$config->{indexer}->{$field}->{isis}}) {
64    
65     my $format = $x->{content};
66 dpavlin 10 my ($s,$d,$i) = (1,1,0); # swish, display default
67 dpavlin 9 $s = 0 if (lc($x->{type}) eq "display");
68     $d = 0 if (lc($x->{type}) eq "swish");
69     ($s,$d,$i) = (0,0,1) if (lc($x->{type}) eq "index");
70     #print STDERR "## s: $s d: $d i: $i ## $format ##\n";
71 dpavlin 3
72 dpavlin 10 $format = $cludge_codepage->convert($format);
73     my ($swish,$display) = parse_format($format,$row);
74     #print STDERR "s: $swish\nd: $display\n" if ($swish);
75 dpavlin 3
76 dpavlin 10 #print STDERR "swish: $swish<-- display: $display<--\n";
77     # FIX: this is ugly, UGLY, cludge: OpenIsis return
78     # UTF8 encoding of strings, but as if source charset
79     # is ISO8859-1 and not some other. This breaks our
80     # isis character encoding, so we convert it first
81     # back to ISO8859-1 (which can actually be different
82     # encoding in isis)
83    
84     $swish_data .= $swish if ($s && $swish);
85     $display_data .= $display if ($d && $display);
86     $index_data .= $display if ($i && $display);
87 dpavlin 3 }
88 dpavlin 9
89 dpavlin 10 #print STDERR "s_d: $swish_data\nd_d: $display_data\n" if ($swish_data);
90     if ($display_data) {
91     $display_data = $isis_codepage->convert($display_data)."##" || $display_data;
92     $xml .= xmlify($field."_display", $display_data);
93     }
94     if ($swish_data) {
95     my $i = Text::Iconv->new($config->{isis_codepage},'ISO8859-2');
96     $swish_data = $i->convert($swish_data);
97     $xml .= xmlify($field."_swish",unac_string('ISO8859-2',$swish_data));
98     #$swish_data = $isis_codepage->convert($swish_data)."##" || $swish_data;
99     #$xml .= xmlify($field."_swish",unac_string($config->{isis_codepage},$swish_data));
100     }
101    
102 dpavlin 9 # index
103     if ($index_data && $index_data ne "") {
104 dpavlin 10 $index_data = $index_codepage->convert($index_data) || $index_data;
105     $index->insert($field, $index_data, $db_dir);
106 dpavlin 9 }
107    
108 dpavlin 3 }
109     if ($xml) {
110 dpavlin 10 #print STDERR "x: $xml\n";
111     return "<xml>\n$xml</xml>\n";
112 dpavlin 3 } else {
113     return;
114     }
115     }
116    
117     ##########################################################################
118    
119 dpavlin 1 my $last_tell=0;
120    
121     my @isis_dirs = ( '.' ); # use dirname as database name
122    
123     if ($opts{m}) {
124     @isis_dirs = split(/,/,$opts{m});
125     }
126    
127     my @isis_dbs;
128    
129     foreach (@isis_dirs) {
130 dpavlin 3 if (-e $config->{isis_data}."/$db_dir/$_/LIBRI") {
131     push @isis_dbs,$config->{isis_data}."/$db_dir/$_/LIBRI/LIBRI";
132 dpavlin 1 }
133 dpavlin 3 if (-e $config->{isis_data}."/$db_dir/$_/PERI") {
134     push @isis_dbs,$config->{isis_data}."/$db_dir/$_/PERI/PERI";
135 dpavlin 1 }
136 dpavlin 3 if (-e $config->{isis_data}."/$db_dir/$_/AMS") {
137     push @isis_dbs,$config->{isis_data}."/$db_dir/$_/AMS/AMS";
138 dpavlin 1 }
139 dpavlin 3 if (-e $config->{isis_data}."/$db_dir/$_/ARTI") {
140     # push @isis_dbs,$config->{isis_data}."/$db_dir/$_/ARTI/ARTI";
141 dpavlin 1 }
142     }
143    
144 dpavlin 3 print STDERR "FATAL: Can't find isis database.\nPerhaps isis_data (".$config->{isis_data}.") has wrong value?\n" if (! @isis_dbs);
145    
146     my $db;
147    
148 dpavlin 1 foreach my $isis_db (@isis_dbs) {
149    
150    
151 dpavlin 3 my $db = OpenIsis::open( $isis_db );
152     if (0) {
153     # # FIX
154     # if (! $db ) {
155     print STDERR "WARNING: can't open '$isis_db'\n";
156     next ;
157     }
158    
159 dpavlin 1 my $max_rowid = OpenIsis::maxRowid( $db );
160    
161 dpavlin 3 print STDERR "Reading database: $isis_db [$max_rowid rows]\n";
162 dpavlin 1
163 dpavlin 10 my $path = $isis_db;
164     $path =~ s#$config->{isis_data}/*##g;
165    
166 dpavlin 3 my $last_p = 0;
167    
168 dpavlin 10 # { my $row_id = 4514;
169 dpavlin 3 # FIX
170 dpavlin 1 for (my $row_id = 1; $row_id <= $max_rowid; $row_id++ ) {
171     my $row = OpenIsis::read( $db, $row_id );
172 dpavlin 3 if ($row && $row->{mfn}) {
173 dpavlin 10 #print STDERR "mfn: ",$row->{mfn},"\n";
174 dpavlin 3 # output current process indicator
175     my $p = int($row->{mfn} * 100 / $max_rowid);
176     if ($p != $last_p) {
177     printf STDERR ("%5d / %5d [%-51s] %-2d %% \r",$row->{mfn},$max_rowid,"=" x ($p/2).">", $p ) if (! $opts{q});
178     $last_p = $p;
179     }
180    
181     if (my $xml = isis2xml($row)) {
182 dpavlin 10 #print STDERR "--ret-->$xml\n";
183     print "Path-Name: $path#".int($row->{mfn})."\n";
184     print "Content-Length: ".(length($xml)+1)."\n";
185     print "Document-Type: XML\n\n$xml\n";
186 dpavlin 3 }
187 dpavlin 1 }
188     }
189 dpavlin 3 print STDERR "\n";
190 dpavlin 1 }
191 dpavlin 3
192 dpavlin 10 # call this to commit index
193     $index->close;
194 dpavlin 3
195     1;
196     __END__
197     ##########################################################################
198    
199     =head1 NAME
200    
201     isis2xml.pl - read isis file and dump XML
202    
203     =head1 DESCRIPTION
204    
205     This command will read ISIS data file using OpenIsis perl module and
206     create XML file for usage with I<SWISH-E>
207     indexer. Dispite it's name, this script B<isn't general xml generator>
208     from isis files (isis allready has something like that). Output of this
209     script is tailor-made for SWISH-E.
210    
211     =head1 AUTHOR
212    
213     Dobrica Pavlinusic <dpavlin@rot13.org>
214    
215     =head1 COPYRIGHT
216    
217     GNU Public License (GPL) v2 or later
218    
219     =head1 SEE ALSO
220    
221     SWISH-E web site at http://www.swish-e.org
222    
223     =cut

Properties

Name Value
cvs2svn:cvs-rev 1.7
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26