19 |
use File::Path; |
use File::Path; |
20 |
use Time::HiRes qw/time/; |
use Time::HiRes qw/time/; |
21 |
use File::Slurp; |
use File::Slurp; |
22 |
|
use MARC::Record 2.0; # need 2.0 for utf-8 encoding see marcpm.sf.net |
23 |
|
|
24 |
=head1 NAME |
=head1 NAME |
25 |
|
|
221 |
%{ $input }, |
%{ $input }, |
222 |
); |
); |
223 |
|
|
224 |
my $rules; |
my @norm_array = ref($input->{normalize}) eq 'ARRAY' ? |
225 |
my $normalize_path = $input->{normalize}->{path}; |
@{ $input->{normalize} } : ( $input->{normalize} ); |
226 |
|
|
227 |
$log->logdie("Found '$normalize_path' as normalization file which isn't supported any more!") unless ( $normalize_path =~ m!\.pl$!i ); |
foreach my $normalize (@norm_array) { |
228 |
|
|
229 |
my $rules = read_file( $normalize_path ) or die "can't open $normalize_path: $!"; |
my $normalize_path = $normalize->{path} || $log->logdie("can't find normalize path in config"); |
230 |
|
|
231 |
foreach my $pos ( 0 ... $input_db->size ) { |
$log->logdie("Found '$normalize_path' as normalization file which isn't supported any more!") unless ( $normalize_path =~ m!\.pl$!i ); |
232 |
|
|
233 |
my $row = $input_db->fetch || next; |
my $rules = read_file( $normalize_path ) or die "can't open $normalize_path: $!"; |
234 |
|
|
235 |
my $mfn = $row->{'000'}->[0]; |
$log->info("Using $normalize_path for normalization..."); |
236 |
|
|
237 |
if (! $mfn || $mfn !~ m#^\d+$#) { |
my $marc_fh; |
238 |
$log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos"); |
if (my $path = $normalize->{marc21}) { |
239 |
$mfn = $pos; |
open($marc_fh, '>', $path) || |
240 |
push @{ $row->{'000'} }, $pos; |
$log->logdie("can't open MARC output $path: $!"); |
241 |
|
|
242 |
|
$log->info("Creating MARC export file $path\n"); |
243 |
} |
} |
244 |
|
|
245 |
|
# reset position in database |
246 |
|
$input_db->seek(1); |
247 |
|
|
248 |
|
foreach my $pos ( 0 ... $input_db->size ) { |
249 |
|
|
250 |
|
my $row = $input_db->fetch || next; |
251 |
|
|
252 |
|
my $mfn = $row->{'000'}->[0]; |
253 |
|
|
254 |
if ($validate) { |
if (! $mfn || $mfn !~ m#^\d+$#) { |
255 |
my @errors = $validate->validate_errors( $row ); |
$log->warn("record $pos doesn't have valid MFN but '$mfn', using $pos"); |
256 |
$log->error( "MFN $mfn validation errors:\n", join("\n", @errors) ) if (@errors); |
$mfn = $pos; |
257 |
|
push @{ $row->{'000'} }, $pos; |
258 |
|
} |
259 |
|
|
260 |
|
|
261 |
|
if ($validate) { |
262 |
|
my @errors = $validate->validate_errors( $row ); |
263 |
|
$log->error( "MFN $mfn validation errors:\n", join("\n", @errors) ) if (@errors); |
264 |
|
} |
265 |
|
|
266 |
|
|
267 |
|
my $ds = WebPAC::Normalize::data_structure( |
268 |
|
row => $row, |
269 |
|
rules => $rules, |
270 |
|
lookup => $lookup ? $lookup->lookup_hash : undef, |
271 |
|
marc_encoding => 'utf-8', |
272 |
|
); |
273 |
|
|
274 |
|
$db->save_ds( |
275 |
|
id => $mfn, |
276 |
|
ds => $ds, |
277 |
|
prefix => $input->{name}, |
278 |
|
) if ($ds && !$stats); |
279 |
|
|
280 |
|
$indexer->add( |
281 |
|
id => $input->{name} . "/" . $mfn, |
282 |
|
ds => $ds, |
283 |
|
type => $config->{$use_indexer}->{type}, |
284 |
|
) if ($indexer && $ds); |
285 |
|
|
286 |
|
if ($marc_fh) { |
287 |
|
my $marc = new MARC::Record; |
288 |
|
$marc->encoding( 'utf-8' ); |
289 |
|
$marc->add_fields( WebPAC::Normalize::_get_marc_fields() ); |
290 |
|
print $marc_fh $marc->as_usmarc; |
291 |
|
} |
292 |
|
|
293 |
|
$total_rows++; |
294 |
} |
} |
295 |
|
|
296 |
|
$log->info("statistics of fields usage:\n", $input_db->stats) if ($stats); |
|
my $ds = WebPAC::Normalize::data_structure( |
|
|
row => $row, |
|
|
rules => $rules, |
|
|
lookup => $lookup ? $lookup->lookup_hash : undef, |
|
|
); |
|
|
|
|
|
$db->save_ds( |
|
|
id => $mfn, |
|
|
ds => $ds, |
|
|
prefix => $input->{name}, |
|
|
) if ($ds && !$stats); |
|
|
|
|
|
$indexer->add( |
|
|
id => $input->{name} . "/" . $mfn, |
|
|
ds => $ds, |
|
|
type => $config->{$use_indexer}->{type}, |
|
|
) if ($indexer); |
|
297 |
|
|
298 |
$total_rows++; |
# close MARC file |
299 |
} |
close($marc_fh) if ($marc_fh); |
300 |
|
|
301 |
$log->info("statistics of fields usage:\n", $input_db->stats) if ($stats); |
} |
302 |
|
|
303 |
}; |
} |
304 |
|
|
305 |
eval { $indexer->finish } if ($indexer && $indexer->can('finish')); |
eval { $indexer->finish } if ($indexer && $indexer->can('finish')); |
306 |
|
|