--- trunk/all2xml.pl 2003/02/22 21:33:04 17 +++ trunk/all2xml.pl 2003/06/25 12:09:27 56 @@ -8,6 +8,7 @@ use Text::Unaccent 1.02; # 1.01 won't compile on my platform, use Text::Iconv; use Config::IniFiles; +use Encode; $|=1; @@ -18,8 +19,7 @@ my $config; use index_DBI; # there is no other, right now ;-) - -my $index = new index_DBI(); # open index +my $index; my %opts; @@ -31,26 +31,38 @@ getopts('d:m:qs', \%opts); -my $db_dir; +my $path; # this is name of database + +Text::Iconv->raise_error(1); # Conversion errors raise exceptions -#die "usage: $0 -d [database_dir] -m [database1,database2] " if (! %opts); +# this is encoding of all files on disk, including import_xml/*.xml file and +# filter/*.pm files! It will be used to store strings in perl internally! +my $codepage = 'ISO-8859-2'; + +my $utf2cp = Text::Iconv->new('UTF-8',$codepage); +# this function will convert data from XML files to local encoding +sub x { + return $utf2cp->convert($_[0]); +} -#print Dumper($config->{indexer}); -#print "-" x 70,"\n"; +# decode isis/excel or other import codepage +my $import2cp; -Text::Iconv->raise_error(1); # Conversion errors raise exceptions +# outgoing xml must be in UTF-8 +my $cp2utf = Text::Iconv->new($codepage,'UTF-8'); -#my $isis_codepage = Text::Iconv->new($config->{isis_codepage},'UTF8'); -#my $index_codepage = Text::Iconv->new($config->{isis_codepage},$config->{index_codepage}); -my $isis_codepage; -my $index_codepage; -my $cludge_codepage = Text::Iconv->new('UTF8','ISO8859-1'); -my $xml_codepage; +# mapping between data type and tag which specify +# format in XML file +my %type2tag = ( + 'isis' => 'isis', + 'excel' => 'column' +); -sub isis2xml { +sub data2xml { use xmlify; + my $type = shift @_; my $row = shift @_; my $add_xml = shift @_; @@ -62,50 +74,103 @@ my %field_usage; # counter for usage of each field - foreach my $field (keys %{$config->{indexer}}) { + # sort subrouting using order="" attribute + sub by_order { + return 0 if (! $config->{indexer}->{$a}->{order}); + return 0 if (! $config->{indexer}->{$b}->{order}); + + return $config->{indexer}->{$a}->{order} <=> + $config->{indexer}->{$b}->{order} ; + } + + foreach my $field (sort by_order keys %{$config->{indexer}}) { + + $field=x($field); $field_usage{$field}++; my $swish_data = ""; my $display_data = ""; + my $line_delimiter; + + my ($swish,$display); + + my $tag = $type2tag{$type} || die "can't find which tag to use for type $type"; + foreach my $x (@{$config->{indexer}->{$field}->{$tag}}) { - foreach my $x (@{$config->{indexer}->{$field}->{isis}}) { + my $format = x($x->{content}); + my $delimiter = x($x->{delimiter}) || ' '; + + my $repeat_off = 0; # repeatable offset - my $format = $x->{content}; my ($s,$d,$i) = (1,1,0); # swish, display default $s = 0 if (lc($x->{type}) eq "display"); $d = 0 if (lc($x->{type}) eq "swish"); ($s,$d,$i) = (0,0,1) if (lc($x->{type}) eq "index"); -#print STDERR "## s: $s d: $d i: $i ## $format ##\n"; - $format = $cludge_codepage->convert($format); - my ($swish,$display) = parse_format($format,$row); -#print STDERR "s: $swish\nd: $display\n" if ($swish); + # what will separate last line from this one? + if ($display_data && $x->{append} && $x->{append} eq "1") { + $line_delimiter = ' '; + } elsif ($display_data) { + $line_delimiter = '
'; + } -#print STDERR "swish: $swish<-- display: $display<--\n"; - # FIX: this is ugly, UGLY, cludge: OpenIsis return - # UTF8 encoding of strings, but as if source charset - # is ISO8859-1 and not some other. This breaks our - # isis character encoding, so we convert it first - # back to ISO8859-1 (which can actually be different - # encoding in isis) + # init vars so that we go into while... + ($swish,$display) = (1,1); - $swish_data .= $swish if ($s && $swish); - $display_data .= $display if ($d && $display); + if ($swish || $display) { + ($swish,$display) = parse_format($type, $format,$row,$repeat_off++,$import2cp); + # filter="name" ; filter this field through + # filter/[name].pm + my $filter = $x->{filter}; + if ($filter) { + require "filter/".$filter.".pm"; + } + # type="swish" ; field for swish + if ($s && $swish) { + if ($filter) { + no strict 'refs'; + $swish_data .= join(" ",&$filter($swish)); + } else { + $swish_data .= $swish; + } + } - # insert into index - if ($i && $display) { - my $index_data = $index_codepage->convert($display) || $display; - $index->insert($field, $index_data, $db_dir); + # type="display" ; field for display + if ($d && $display) { + if ($line_delimiter && $display_data) { + $display_data .= $line_delimiter; + undef $line_delimiter; + } + if ($filter) { + no strict 'refs'; + $display_data .= join($delimiter,&$filter($display)); + } else { + if ($display_data) { + $display_data .= $delimiter.$display; + } else { + $display_data .= $display; + } + } + } + + # type="index" ; insert into index + if ($i && $display) { + my $index_data = $display; + if ($filter) { + no strict 'refs'; + foreach my $d (&$filter($index_data)) { + $index->insert($field, $d, $path); + } + } else { + $index->insert($field, $index_data, $path); + } + } } } -#print STDERR "s_d: $swish_data\nd_d: $display_data\n" if ($swish_data); if ($display_data) { - $display_data = $isis_codepage->convert($display_data) || die "Can't convert '$display_data' !"; - # FIX: this is removed and replaced by html tag. - #$xml .= xmlify($field."_display", $display_data); if ($field eq "headline") { $xml .= xmlify("headline", $display_data); @@ -117,21 +182,23 @@ $field_name = $config->{indexer}->{$field}->{name_singular}."#-#"; } elsif ($config->{indexer}->{$field}->{name_plural}) { $field_name = $config->{indexer}->{$field}->{name_plural}."#-#"; - } else { + } elsif ($config->{indexer}->{$field}->{name}) { $field_name = $config->{indexer}->{$field}->{name}."#-#"; + } else { + print STDERR "WARNING: field '$field' doesn't have 'name' attribute!"; } if ($field_name) { - $html .= $xml_codepage->convert($field_name); + $html .= x($field_name); } $html .= $display_data."###\n"; } } if ($swish_data) { - my $i = Text::Iconv->new($config->{isis_codepage},'ISO8859-2'); - $swish_data = $i->convert($swish_data); - $xml .= xmlify($field."_swish",unac_string('ISO8859-2',$swish_data)); - #$swish_data = $isis_codepage->convert($swish_data)."##" || $swish_data; - #$xml .= xmlify($field."_swish",unac_string($config->{isis_codepage},$swish_data)); + # remove extra spaces + $swish_data =~ s/ +/ /g; + $swish_data =~ s/ +$//g; + + $xml .= xmlify($field."_swish", unac_string($codepage,$swish_data)); } @@ -143,7 +210,6 @@ } if ($xml) { -#print STDERR "x: $xml\n"; $xml .= $add_xml if ($add_xml); return "\n$xml\n"; } else { @@ -153,58 +219,131 @@ ########################################################################## +# read configuration for this script my $cfg = new Config::IniFiles( -file => $config_file ); +# read global.conf configuration +my $cfg_global = new Config::IniFiles( -file => 'global.conf' ); + +# open index +$index = new index_DBI( + $cfg_global->val('global', 'dbi_dbd'), + $cfg_global->val('global', 'dbi_dsn'), + $cfg_global->val('global', 'dbi_user'), + $cfg_global->val('global', 'dbi_passwd') || '', + ); + foreach my $database ($cfg->Sections) { - my $isis_db = $cfg -> val($database, 'isis_db'); - my $type = $cfg -> val($database, 'type'); - my $add_xml = $cfg -> val($database, 'xml'); + my $type = lc($cfg -> val($database, 'type')) || die "$database doesn't have 'type' defined"; + my $add_xml = $cfg -> val($database, 'xml'); # optional + +print STDERR "reading ./import_xml/$type.xml\n"; - # read configuration for this type - $config=XMLin("./import_xml/$type.xml", forcearray => [ 'isis' ], forcecontent => 1); - $isis_codepage = Text::Iconv->new($config->{isis_codepage},'UTF8'); - $index_codepage = Text::Iconv->new($config->{isis_codepage},$config->{index_codepage}); - $xml_codepage = Text::Iconv->new($cfg->val($database,'xml_codepage'),'UTF8'); + $config=XMLin("./import_xml/$type.xml", forcearray => [ $type2tag{$type} ], forcecontent => 1); - my $db = OpenIsis::open( $isis_db ); - if (0) { -# # FIX -# if (! $db ) { - print STDERR "WARNING: can't open '$isis_db'\n"; - next ; + # output current progress indicator + my $last_p = 0; + sub progress { + #return if (! $opts{q}); # FIXME + my $current = shift; + my $total = shift; + my $p = int($current * 100 / $total); + if ($p != $last_p) { + printf STDERR ("%5d / %5d [%-51s] %-2d %% \r",$current,$total,"=" x ($p/2).">", $p ); + $last_p = $p; + } } - my $max_rowid = OpenIsis::maxRowid( $db ); + # now read database +print STDERR "using: $type...\n"; - print STDERR "Reading database: $isis_db [$max_rowid rows]\n"; + if ($type eq "isis") { + my $isis_db = $cfg -> val($database, 'isis_db') || die "$database doesn't have 'isis_db' defined!"; - my $path = $database; # was $isis_db + $import2cp = Text::Iconv->new($config->{isis_codepage},$codepage); + my $db = OpenIsis::open( $isis_db ); - my $last_p = 0; + my $max_rowid = OpenIsis::maxRowid( $db ); + + print STDERR "Reading database: $isis_db [$max_rowid rows]\n"; + + my $path = $database; + + for (my $row_id = 1; $row_id <= $max_rowid; $row_id++ ) { + my $row = OpenIsis::read( $db, $row_id ); + if ($row && $row->{mfn}) { + + progress($row->{mfn}, $max_rowid); + + my $swishpath = $path."#".int($row->{mfn}); + + if (my $xml = data2xml($type,$row,$add_xml)) { + $xml = $cp2utf->convert($xml); + use bytes; # as opposed to chars + print "Path-Name: $swishpath\n"; + print "Content-Length: ".(length($xml)+1)."\n"; + print "Document-Type: XML\n\n$xml\n"; + } + } + } + print STDERR "\n"; + + } elsif ($type eq "excel") { + use Spreadsheet::ParseExcel; + use Spreadsheet::ParseExcel::Utility qw(int2col); + + $import2cp = Text::Iconv->new($config->{excel_codepage},$codepage); + my $excel_file = $cfg -> val($database, 'excel_file') || die "$database doesn't have 'excel_file' defined!"; + + my $sheet = x($config->{sheet}) || die "no sheet in $type.xml"; + my $start_row = x($config->{start_row}) - 1 || die "no start_row in $type.xml"; + + my $oBook = Spreadsheet::ParseExcel::Workbook->Parse($excel_file) || die "can't open Excel file '$excel_file'"; + + my $sheet_nr = 0; + foreach my $oWks (@{$oBook->{Worksheet}}) { + #print STDERR "-- SHEET $sheet_nr:", $oWks->{Name}, "\n"; + last if ($oWks->{Name} eq $sheet); + $sheet_nr++; + } + + my $oWorksheet = $oBook->{Worksheet}[$sheet_nr]; + + print STDERR "using sheet: ",$oWorksheet->{Name},"\n"; + defined ($oWorksheet) || die "can't find sheet '$sheet' in $excel_file"; + my $end_row = x($config->{end_row}) || $oWorksheet->{MaxRow}; + + for(my $iR = $start_row ; defined $end_row && $iR <= $end_row ; $iR++) { + my $row; + for(my $iC = $oWorksheet->{MinCol} ; defined $oWorksheet->{MaxCol} && $iC <= $oWorksheet->{MaxCol} ; $iC++) { + my $cell = $oWorksheet->{Cells}[$iR][$iC]; + if ($cell) { + $row->{int2col($iC)} = $cell->Value; + } + } + + progress($iR, $end_row); -# { my $row_id = 4514; -# FIX - for (my $row_id = 1; $row_id <= $max_rowid; $row_id++ ) { - my $row = OpenIsis::read( $db, $row_id ); - if ($row && $row->{mfn}) { -#print STDERR "mfn: ",$row->{mfn},"\n"; - # output current process indicator - my $p = int($row->{mfn} * 100 / $max_rowid); - if ($p != $last_p) { - printf STDERR ("%5d / %5d [%-51s] %-2d %% \r",$row->{mfn},$max_rowid,"=" x ($p/2).">", $p ) if (! $opts{q}); - $last_p = $p; - } - - if (my $xml = isis2xml($row,$add_xml)) { -#print STDERR "--ret-->$xml\n"; - print "Path-Name: $path#".int($row->{mfn})."\n"; +# print "row[$iR/$end_row] "; +# foreach (keys %{$row}) { +# print "$_: ",$row->{$_},"\t"; +# } +# print "\n"; + + my $swishpath = $database."#".$iR; + + next if (! $row); + + if (my $xml = data2xml($type,$row,$add_xml)) { + $xml = $cp2utf->convert($xml); + use bytes; # as opposed to chars + print "Path-Name: $swishpath\n"; print "Content-Length: ".(length($xml)+1)."\n"; print "Document-Type: XML\n\n$xml\n"; } } } - print STDERR "\n"; } # call this to commit index