--- trunk/all2xml.pl 2003/01/16 17:35:54 10 +++ trunk/all2xml.pl 2003/02/22 21:33:04 17 @@ -7,13 +7,15 @@ use XML::Simple; use Text::Unaccent 1.02; # 1.01 won't compile on my platform, use Text::Iconv; - +use Config::IniFiles; $|=1; -my $config; +my $config_file = $0; +$config_file =~ s/\.pl$/.conf/; +die "FATAL: can't find configuration file '$config_file'" if (! -e $config_file); -$config=XMLin(undef, forcearray => [ 'isis' ], forcecontent => 1); +my $config; use index_DBI; # there is no other, right now ;-) @@ -29,7 +31,7 @@ getopts('d:m:qs', \%opts); -my $db_dir = $opts{d} || "ps"; # FIX +my $db_dir; #die "usage: $0 -d [database_dir] -m [database1,database2] " if (! %opts); @@ -38,27 +40,34 @@ Text::Iconv->raise_error(1); # Conversion errors raise exceptions -my $isis_codepage = Text::Iconv->new($config->{isis_codepage},'UTF8'); -my $index_codepage = Text::Iconv->new($config->{isis_codepage},$config->{index_codepage}); +#my $isis_codepage = Text::Iconv->new($config->{isis_codepage},'UTF8'); +#my $index_codepage = Text::Iconv->new($config->{isis_codepage},$config->{index_codepage}); +my $isis_codepage; +my $index_codepage; my $cludge_codepage = Text::Iconv->new('UTF8','ISO8859-1'); +my $xml_codepage; sub isis2xml { use xmlify; my $row = shift @_; + my $add_xml = shift @_; my $xml; - $xml .= xmlify('db_dir',$db_dir); # FIX remove? use parse_format; + my $html = ""; # html formatted display output + + my %field_usage; # counter for usage of each field foreach my $field (keys %{$config->{indexer}}) { + $field_usage{$field}++; + my $swish_data = ""; my $display_data = ""; - my $index_data = ""; foreach my $x (@{$config->{indexer}->{$field}->{isis}}) { @@ -83,13 +92,39 @@ $swish_data .= $swish if ($s && $swish); $display_data .= $display if ($d && $display); - $index_data .= $display if ($i && $display); + + # insert into index + if ($i && $display) { + my $index_data = $index_codepage->convert($display) || $display; + $index->insert($field, $index_data, $db_dir); + } } + #print STDERR "s_d: $swish_data\nd_d: $display_data\n" if ($swish_data); if ($display_data) { - $display_data = $isis_codepage->convert($display_data)."##" || $display_data; - $xml .= xmlify($field."_display", $display_data); + $display_data = $isis_codepage->convert($display_data) || die "Can't convert '$display_data' !"; + # FIX: this is removed and replaced by html tag. + #$xml .= xmlify($field."_display", $display_data); + + if ($field eq "headline") { + $xml .= xmlify("headline", $display_data); + } else { + + # find field name (signular, plural) + my $field_name = ""; + if ($config->{indexer}->{$field}->{name_singular} && $field_usage{$field} == 1) { + $field_name = $config->{indexer}->{$field}->{name_singular}."#-#"; + } elsif ($config->{indexer}->{$field}->{name_plural}) { + $field_name = $config->{indexer}->{$field}->{name_plural}."#-#"; + } else { + $field_name = $config->{indexer}->{$field}->{name}."#-#"; + } + if ($field_name) { + $html .= $xml_codepage->convert($field_name); + } + $html .= $display_data."###\n"; + } } if ($swish_data) { my $i = Text::Iconv->new($config->{isis_codepage},'ISO8859-2'); @@ -99,15 +134,17 @@ #$xml .= xmlify($field."_swish",unac_string($config->{isis_codepage},$swish_data)); } - # index - if ($index_data && $index_data ne "") { - $index_data = $index_codepage->convert($index_data) || $index_data; - $index->insert($field, $index_data, $db_dir); - } } + + # dump formatted output in + if ($html) { + $xml .= xmlify("html",$html); + } + if ($xml) { #print STDERR "x: $xml\n"; + $xml .= $add_xml if ($add_xml); return "\n$xml\n"; } else { return; @@ -116,37 +153,19 @@ ########################################################################## -my $last_tell=0; - -my @isis_dirs = ( '.' ); # use dirname as database name - -if ($opts{m}) { - @isis_dirs = split(/,/,$opts{m}); -} - -my @isis_dbs; - -foreach (@isis_dirs) { - if (-e $config->{isis_data}."/$db_dir/$_/LIBRI") { - push @isis_dbs,$config->{isis_data}."/$db_dir/$_/LIBRI/LIBRI"; - } - if (-e $config->{isis_data}."/$db_dir/$_/PERI") { - push @isis_dbs,$config->{isis_data}."/$db_dir/$_/PERI/PERI"; - } - if (-e $config->{isis_data}."/$db_dir/$_/AMS") { - push @isis_dbs,$config->{isis_data}."/$db_dir/$_/AMS/AMS"; - } - if (-e $config->{isis_data}."/$db_dir/$_/ARTI") { -# push @isis_dbs,$config->{isis_data}."/$db_dir/$_/ARTI/ARTI"; - } -} - -print STDERR "FATAL: Can't find isis database.\nPerhaps isis_data (".$config->{isis_data}.") has wrong value?\n" if (! @isis_dbs); - -my $db; +my $cfg = new Config::IniFiles( -file => $config_file ); -foreach my $isis_db (@isis_dbs) { +foreach my $database ($cfg->Sections) { + my $isis_db = $cfg -> val($database, 'isis_db'); + my $type = $cfg -> val($database, 'type'); + my $add_xml = $cfg -> val($database, 'xml'); + + # read configuration for this type + $config=XMLin("./import_xml/$type.xml", forcearray => [ 'isis' ], forcecontent => 1); + $isis_codepage = Text::Iconv->new($config->{isis_codepage},'UTF8'); + $index_codepage = Text::Iconv->new($config->{isis_codepage},$config->{index_codepage}); + $xml_codepage = Text::Iconv->new($cfg->val($database,'xml_codepage'),'UTF8'); my $db = OpenIsis::open( $isis_db ); if (0) { @@ -160,8 +179,7 @@ print STDERR "Reading database: $isis_db [$max_rowid rows]\n"; - my $path = $isis_db; - $path =~ s#$config->{isis_data}/*##g; + my $path = $database; # was $isis_db my $last_p = 0; @@ -178,7 +196,7 @@ $last_p = $p; } - if (my $xml = isis2xml($row)) { + if (my $xml = isis2xml($row,$add_xml)) { #print STDERR "--ret-->$xml\n"; print "Path-Name: $path#".int($row->{mfn})."\n"; print "Content-Length: ".(length($xml)+1)."\n";