/[simile]/links/csv2js.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /links/csv2js.pl

Parent Directory Parent Directory | Revision Log Revision Log


Revision 5 - (hide annotations)
Thu Aug 23 12:00:29 2007 UTC (16 years, 7 months ago) by dpavlin
File MIME type: text/plain
File size: 3687 byte(s)
convert strings that look like kb/mb/gb to human readable,
unroll every splited value into it's own field
1 dpavlin 1 #!/usr/bin/perl
2    
3     use warnings;
4     use strict;
5    
6     use Text::CSV_XS;
7     use Text::CSV::Separator qw(get_separator);
8     use Carp qw/confess/;
9     use LWP::Simple;
10 dpavlin 5 use Number::Bytes::Human qw/format_bytes/;
11 dpavlin 1
12     use JSON::Syck;
13     use Data::Dump qw/dump/;
14    
15     $|++;
16    
17     my $csv_path = 'links.csv';
18     my $img_path = 'img';
19     my $first_line_labels = 1;
20     my $split_fields = {
21     label => sub { return split(/,\s*/,$_[0]) },
22     };
23    
24     my @char_list = get_separator( path => $csv_path );
25    
26     my $separator;
27     if (@char_list) {
28     if (@char_list == 1) {
29     $separator = $char_list[0];
30     } else {
31     $separator = $char_list[0];
32     }
33     } else {
34     die "Couldn't detect the field separator.\n";
35     }
36    
37     warn "Separator: $separator\n";
38    
39     my $csv_parser = Text::CSV_XS->new({
40     sep_char => $separator,
41     # binary => '1',
42     # always_quote => '1'
43     });
44    
45     open my $csv_fh, '<', $csv_path;
46    
47     my @dump;
48    
49     my @labels;
50    
51     my $split_stats;
52    
53     my $multiplier = {
54     kb => 1024,
55 dpavlin 5 mb => 1024 * 1024,
56 dpavlin 1 gb => 1024 * 1024 * 1024,
57     };
58    
59     my $multiplier_regex = join('|',keys %$multiplier);
60    
61     sub clean {
62     my @out;
63     foreach my $l ( @_ ) {
64     my $o = $l;
65     $l =~ s/^(['"])(.*)\1/$2/;
66     $l =~ s/^\s+//s;
67     $l =~ s/\s+$//s;
68     push @out, $l;
69     warn "clean '$o' -> '$l'\n" if ( $o ne $l );
70     }
71     return @out if wantarray;
72     return shift @out;
73     }
74    
75 dpavlin 5 sub human {
76     my $s = shift;
77    
78     if ( $s =~ m/(\d+)\s*($multiplier_regex)/i) {
79     my ( $v, $m ) = ( $1, lc($2) );
80     my $factor = $multiplier->{$m};
81     confess "can't find multiplier $m" unless defined $factor;
82     my $new = format_bytes( $v * $factor, bs => 1024 );
83     warn "## [$s] $v * $factor ($m) -> $new\n";
84     return $new;
85     }
86     return;
87     }
88    
89 dpavlin 1 while (<$csv_fh>) {
90     $csv_parser->parse($_);
91     my @fields = $csv_parser->fields;
92    
93     if ( $first_line_labels && $. == 1 ) {
94     @labels = @fields;
95     next;
96     }
97    
98     my $h;
99     foreach my $i ( 0 .. $#fields ) {
100     my $l = $labels[$i];
101     die "no label for field $i '$fields[$i]'" unless $l;
102    
103     my $v = clean( $fields[$i] );
104     # FIXME reject some values?
105    
106     $h->{ $l } = $v;
107    
108     if ( my $split = $split_fields->{$l} ) {
109     confess "expected CODE for \$split_files->{$l}" unless ref($split) eq 'CODE';
110    
111     my @sv = $split->( $v );
112    
113     # warn "sv = ",dump( @sv );
114    
115     foreach my $j ( 0 .. $#sv ) {
116    
117     my $v = clean( $sv[$j] );
118    
119     if ( $j == 0 ) {
120     $h->{ $l . '_short' } = $v;
121     }
122    
123 dpavlin 5 if ( my $human = human( $v ) ) {
124     $h->{ $l . '_' . $j . '_human' } = $human;
125     } else {
126     $h->{ $l . '_' . $j } = $v;
127 dpavlin 1 }
128    
129     $split_stats->{$v}->{$j}++;
130     $split_stats->{$v}->{sum}++;
131     push @{ $split_stats->{$v}->{rec}->{$#dump + 1} }, $j;
132     }
133     }
134     }
135     warn "\nRecord #$. ",dump($h),"\n";
136    
137     my $id = $h->{id};
138    
139     if ( ! defined($id) || $id eq '' ) {
140     warn "## skipped: $_";
141     next;
142     }
143    
144     my $url = "http://www.links.hr/photo/big/$id.jpg";
145     my $img_thumb_path = "$img_path/t/$id.jpg";
146     my $img_orig_path = "$img_path/$id.jpg";
147    
148     if ( mirror( $url, $img_orig_path ) != RC_NOT_MODIFIED ) {
149     warn "$url -> $img_orig_path\n";
150     }
151 dpavlin 2 system('convert', '-geometry', '320x200', $img_orig_path, $img_thumb_path ) if ! -e $img_thumb_path;
152 dpavlin 1
153     $h->{'image-url'} = $img_orig_path;
154     $h->{'image-thumb-url'} = $img_thumb_path;
155    
156     push @dump, $h;
157     }
158    
159     close $csv_fh;
160    
161     foreach my $v ( keys %$split_stats ) {
162    
163     if ( $split_stats->{$v}->{sum} == 1 ) {
164     delete( $split_stats->{$v} );
165     next;
166     }
167    
168     foreach my $i ( keys %{ $split_stats->{$v}->{rec} } ) {
169     push @{ $dump[ $i ]->{feature} }, $v;
170     }
171     }
172    
173     #warn "split_stats = ", dump( $split_stats ), "\n";
174    
175     warn "dump = ", dump( @dump ), "\n";
176    
177     print "features: .", join(', .', keys %$split_stats), "\n";
178    
179     my $js_path = $csv_path;
180     $js_path =~ s/\.csv/.js/gi;
181    
182     open my $fh, '>', $js_path || die "can't open $js_path: $!";
183     print $fh JSON::Syck::Dump( { items => \@dump } );
184     close $fh;
185    

Properties

Name Value
svn:executable

  ViewVC Help
Powered by ViewVC 1.1.26