/[webpac]/branches/ffzg/all2xml.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /branches/ffzg/all2xml.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 57 - (show annotations)
Fri Jul 4 15:05:23 2003 UTC (20 years, 8 months ago) by dpavlin
Original Path: trunk/all2xml.pl
File MIME type: text/plain
File size: 9738 byte(s)
don't choke on input which iconv can't convert

1 #!/usr/bin/perl -w
2
3 use strict;
4 use OpenIsis;
5 use Getopt::Std;
6 use Data::Dumper;
7 use XML::Simple;
8 use Text::Unaccent 1.02; # 1.01 won't compile on my platform,
9 use Text::Iconv;
10 use Config::IniFiles;
11 use Encode;
12
13 $|=1;
14
15 my $config_file = $0;
16 $config_file =~ s/\.pl$/.conf/;
17 die "FATAL: can't find configuration file '$config_file'" if (! -e $config_file);
18
19 my $config;
20
21 use index_DBI; # there is no other, right now ;-)
22 my $index;
23
24 my %opts;
25
26 # usage:
27 # -d directory name
28 # -m multiple directories
29 # -q quiet
30 # -s run swish
31
32 getopts('d:m:qs', \%opts);
33
34 my $path; # this is name of database
35
36 Text::Iconv->raise_error(0); # Conversion errors don't raise exceptions
37
38 # this is encoding of all files on disk, including import_xml/*.xml file and
39 # filter/*.pm files! It will be used to store strings in perl internally!
40 my $codepage = 'ISO-8859-2';
41
42 my $utf2cp = Text::Iconv->new('UTF-8',$codepage);
43 # this function will convert data from XML files to local encoding
44 sub x {
45 return $utf2cp->convert($_[0]);
46 }
47
48 # decode isis/excel or other import codepage
49 my $import2cp;
50
51 # outgoing xml must be in UTF-8
52 my $cp2utf = Text::Iconv->new($codepage,'UTF-8');
53
54 # mapping between data type and tag which specify
55 # format in XML file
56 my %type2tag = (
57 'isis' => 'isis',
58 'excel' => 'column'
59 );
60
61 sub data2xml {
62
63 use xmlify;
64
65 my $type = shift @_;
66 my $row = shift @_;
67 my $add_xml = shift @_;
68
69 my $xml;
70
71 use parse_format;
72
73 my $html = ""; # html formatted display output
74
75 my %field_usage; # counter for usage of each field
76
77 # sort subrouting using order="" attribute
78 sub by_order {
79 return 0 if (! $config->{indexer}->{$a}->{order});
80 return 0 if (! $config->{indexer}->{$b}->{order});
81
82 return $config->{indexer}->{$a}->{order} <=>
83 $config->{indexer}->{$b}->{order} ;
84 }
85
86 foreach my $field (sort by_order keys %{$config->{indexer}}) {
87
88 $field=x($field);
89
90 $field_usage{$field}++;
91
92 my $swish_data = "";
93 my $display_data = "";
94 my $line_delimiter;
95
96 my ($swish,$display);
97
98 my $tag = $type2tag{$type} || die "can't find which tag to use for type $type";
99 foreach my $x (@{$config->{indexer}->{$field}->{$tag}}) {
100
101 my $format = x($x->{content});
102 my $delimiter = x($x->{delimiter}) || ' ';
103
104 my $repeat_off = 0; # repeatable offset
105
106 my ($s,$d,$i) = (1,1,0); # swish, display default
107 $s = 0 if (lc($x->{type}) eq "display");
108 $d = 0 if (lc($x->{type}) eq "swish");
109 ($s,$d,$i) = (0,0,1) if (lc($x->{type}) eq "index");
110
111 # what will separate last line from this one?
112 if ($display_data && $x->{append} && $x->{append} eq "1") {
113 $line_delimiter = ' ';
114 } elsif ($display_data) {
115 $line_delimiter = '<br/>';
116 }
117
118 # init vars so that we go into while...
119 ($swish,$display) = (1,1);
120
121 if ($swish || $display) {
122 ($swish,$display) = parse_format($type, $format,$row,$repeat_off++,$import2cp);
123 # filter="name" ; filter this field through
124 # filter/[name].pm
125 my $filter = $x->{filter};
126 if ($filter) {
127 require "filter/".$filter.".pm";
128 }
129 # type="swish" ; field for swish
130 if ($s && $swish) {
131 if ($filter) {
132 no strict 'refs';
133 $swish_data .= join(" ",&$filter($swish));
134 } else {
135 $swish_data .= $swish;
136 }
137 }
138
139 # type="display" ; field for display
140 if ($d && $display) {
141 if ($line_delimiter && $display_data) {
142 $display_data .= $line_delimiter;
143 undef $line_delimiter;
144 }
145 if ($filter) {
146 no strict 'refs';
147 $display_data .= join($delimiter,&$filter($display));
148 } else {
149 if ($display_data) {
150 $display_data .= $delimiter.$display;
151 } else {
152 $display_data .= $display;
153 }
154 }
155 }
156
157 # type="index" ; insert into index
158 if ($i && $display) {
159 my $index_data = $display;
160 if ($filter) {
161 no strict 'refs';
162 foreach my $d (&$filter($index_data)) {
163 $index->insert($field, $d, $path);
164 }
165 } else {
166 $index->insert($field, $index_data, $path);
167 }
168 }
169 }
170 }
171
172
173 if ($display_data) {
174
175 if ($field eq "headline") {
176 $xml .= xmlify("headline", $display_data);
177 } else {
178
179 # find field name (signular, plural)
180 my $field_name = "";
181 if ($config->{indexer}->{$field}->{name_singular} && $field_usage{$field} == 1) {
182 $field_name = $config->{indexer}->{$field}->{name_singular}."#-#";
183 } elsif ($config->{indexer}->{$field}->{name_plural}) {
184 $field_name = $config->{indexer}->{$field}->{name_plural}."#-#";
185 } elsif ($config->{indexer}->{$field}->{name}) {
186 $field_name = $config->{indexer}->{$field}->{name}."#-#";
187 } else {
188 print STDERR "WARNING: field '$field' doesn't have 'name' attribute!";
189 }
190 if ($field_name) {
191 $html .= x($field_name);
192 }
193 $html .= $display_data."###\n";
194 }
195 }
196 if ($swish_data) {
197 # remove extra spaces
198 $swish_data =~ s/ +/ /g;
199 $swish_data =~ s/ +$//g;
200
201 $xml .= xmlify($field."_swish", unac_string($codepage,$swish_data));
202 }
203
204
205 }
206
207 # dump formatted output in <html>
208 if ($html) {
209 $xml .= xmlify("html",$html);
210 }
211
212 if ($xml) {
213 $xml .= $add_xml if ($add_xml);
214 return "<xml>\n$xml</xml>\n";
215 } else {
216 return;
217 }
218 }
219
220 ##########################################################################
221
222 # read configuration for this script
223 my $cfg = new Config::IniFiles( -file => $config_file );
224
225 # read global.conf configuration
226 my $cfg_global = new Config::IniFiles( -file => 'global.conf' );
227
228 # open index
229 $index = new index_DBI(
230 $cfg_global->val('global', 'dbi_dbd'),
231 $cfg_global->val('global', 'dbi_dsn'),
232 $cfg_global->val('global', 'dbi_user'),
233 $cfg_global->val('global', 'dbi_passwd') || '',
234 );
235
236 foreach my $database ($cfg->Sections) {
237
238 my $type = lc($cfg -> val($database, 'type')) || die "$database doesn't have 'type' defined";
239 my $add_xml = $cfg -> val($database, 'xml'); # optional
240
241 print STDERR "reading ./import_xml/$type.xml\n";
242
243 $config=XMLin("./import_xml/$type.xml", forcearray => [ $type2tag{$type} ], forcecontent => 1);
244
245 # output current progress indicator
246 my $last_p = 0;
247 sub progress {
248 #return if (! $opts{q}); # FIXME
249 my $current = shift;
250 my $total = shift || 1;
251 my $p = int($current * 100 / $total);
252 if ($p != $last_p) {
253 printf STDERR ("%5d / %5d [%-51s] %-2d %% \r",$current,$total,"=" x ($p/2).">", $p );
254 $last_p = $p;
255 }
256 }
257
258 # now read database
259 print STDERR "using: $type...\n";
260
261 if ($type eq "isis") {
262 my $isis_db = $cfg -> val($database, 'isis_db') || die "$database doesn't have 'isis_db' defined!";
263
264 $import2cp = Text::Iconv->new($config->{isis_codepage},$codepage);
265 my $db = OpenIsis::open( $isis_db );
266
267 my $max_rowid = OpenIsis::maxRowid( $db );
268
269 print STDERR "Reading database: $isis_db [$max_rowid rows]\n";
270
271 my $path = $database;
272
273 for (my $row_id = 1; $row_id <= $max_rowid; $row_id++ ) {
274 my $row = OpenIsis::read( $db, $row_id );
275 if ($row && $row->{mfn}) {
276
277 progress($row->{mfn}, $max_rowid);
278
279 my $swishpath = $path."#".int($row->{mfn});
280
281 if (my $xml = data2xml($type,$row,$add_xml)) {
282 $xml = $cp2utf->convert($xml);
283 use bytes; # as opposed to chars
284 print "Path-Name: $swishpath\n";
285 print "Content-Length: ".(length($xml)+1)."\n";
286 print "Document-Type: XML\n\n$xml\n";
287 }
288 }
289 }
290 print STDERR "\n";
291
292 } elsif ($type eq "excel") {
293 use Spreadsheet::ParseExcel;
294 use Spreadsheet::ParseExcel::Utility qw(int2col);
295
296 $import2cp = Text::Iconv->new($config->{excel_codepage},$codepage);
297 my $excel_file = $cfg -> val($database, 'excel_file') || die "$database doesn't have 'excel_file' defined!";
298
299 my $sheet = x($config->{sheet}) || die "no sheet in $type.xml";
300 my $start_row = x($config->{start_row}) - 1 || die "no start_row in $type.xml";
301
302 my $oBook = Spreadsheet::ParseExcel::Workbook->Parse($excel_file) || die "can't open Excel file '$excel_file'";
303
304 my $sheet_nr = 0;
305 foreach my $oWks (@{$oBook->{Worksheet}}) {
306 #print STDERR "-- SHEET $sheet_nr:", $oWks->{Name}, "\n";
307 last if ($oWks->{Name} eq $sheet);
308 $sheet_nr++;
309 }
310
311 my $oWorksheet = $oBook->{Worksheet}[$sheet_nr];
312
313 print STDERR "using sheet: ",$oWorksheet->{Name},"\n";
314 defined ($oWorksheet) || die "can't find sheet '$sheet' in $excel_file";
315 my $end_row = x($config->{end_row}) || $oWorksheet->{MaxRow};
316
317 for(my $iR = $start_row ; defined $end_row && $iR <= $end_row ; $iR++) {
318 my $row;
319 for(my $iC = $oWorksheet->{MinCol} ; defined $oWorksheet->{MaxCol} && $iC <= $oWorksheet->{MaxCol} ; $iC++) {
320 my $cell = $oWorksheet->{Cells}[$iR][$iC];
321 if ($cell) {
322 $row->{int2col($iC)} = $cell->Value;
323 }
324 }
325
326 progress($iR, $end_row);
327
328 # print "row[$iR/$end_row] ";
329 # foreach (keys %{$row}) {
330 # print "$_: ",$row->{$_},"\t";
331 # }
332 # print "\n";
333
334 my $swishpath = $database."#".$iR;
335
336 next if (! $row);
337
338 if (my $xml = data2xml($type,$row,$add_xml)) {
339 $xml = $cp2utf->convert($xml);
340 use bytes; # as opposed to chars
341 print "Path-Name: $swishpath\n";
342 print "Content-Length: ".(length($xml)+1)."\n";
343 print "Document-Type: XML\n\n$xml\n";
344 }
345 }
346 }
347 }
348
349 # call this to commit index
350 $index->close;
351
352 1;
353 __END__
354 ##########################################################################
355
356 =head1 NAME
357
358 isis2xml.pl - read isis file and dump XML
359
360 =head1 DESCRIPTION
361
362 This command will read ISIS data file using OpenIsis perl module and
363 create XML file for usage with I<SWISH-E>
364 indexer. Dispite it's name, this script B<isn't general xml generator>
365 from isis files (isis allready has something like that). Output of this
366 script is tailor-made for SWISH-E.
367
368 =head1 AUTHOR
369
370 Dobrica Pavlinusic <dpavlin@rot13.org>
371
372 =head1 COPYRIGHT
373
374 GNU Public License (GPL) v2 or later
375
376 =head1 SEE ALSO
377
378 SWISH-E web site at http://www.swish-e.org
379
380 =cut

Properties

Name Value
cvs2svn:cvs-rev 1.23
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26