13 |
use File::Path qw(make_path remove_tree); |
use File::Path qw(make_path remove_tree); |
14 |
use MongoDB; |
use MongoDB; |
15 |
|
|
16 |
my $name = 'pxelator'; |
my $name = shift @ARGV || 'pxelator'; |
17 |
|
|
18 |
my $conn = MongoDB::Connection->new; |
my $conn = MongoDB::Connection->new; |
19 |
my $db = $conn->get_database( $name ); |
my $db = $conn->get_database( $name ); |
46 |
|
|
47 |
my $total = <$sock>; |
my $total = <$sock>; |
48 |
$total =~ s{^.*total_rows\D+(\d+).+$}{$1}; |
$total =~ s{^.*total_rows\D+(\d+).+$}{$1}; |
49 |
warn "# total: $total\n"; |
|
50 |
|
$|=1; |
51 |
|
print "# $name total: $total\n"; |
52 |
|
|
53 |
|
my $start_t = time(); |
54 |
|
my $count = 0; |
55 |
|
|
56 |
|
|
57 |
while(<$sock>) { |
while(<$sock>) { |
58 |
if ( /"id":"([^"]+)"/ ) { |
if ( /"id":"([^"]+)"/ ) { |
60 |
s/,[\r\n]+$//; # cleanup JSON |
s/,[\r\n]+$//; # cleanup JSON |
61 |
my $json = from_json( $_ ); |
my $json = from_json( $_ ); |
62 |
$audit->insert( $json->{doc} ); |
$audit->insert( $json->{doc} ); |
63 |
print STDERR "."; |
printf " %d/%d %.2f%% %.2f/s\r", $count, $total |
64 |
|
, ( $count / $total ) * 100 |
65 |
|
, ( $count / ( time() - $start_t ) ) |
66 |
|
if ++$count % 1000 == 0; |
67 |
} else { |
} else { |
68 |
warn "UNKNOWN: $_"; |
warn "UNKNOWN: $_"; |
69 |
} |
} |