/[Z3950-HTML-Scraper]/COBISS.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /COBISS.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 14 - (show annotations)
Fri Oct 22 21:31:08 2010 UTC (13 years, 5 months ago) by dpavlin
File size: 3662 byte(s)
make usemap configurable

1 package COBISS;
2
3 use warnings;
4 use strict;
5
6 use WWW::Mechanize;
7 use MARC::Record;
8 use Data::Dump qw/dump/;
9
10 binmode STDOUT, ':utf8';
11
12 my $cobiss_marc21 = {
13 '010' => { a => [ '020', 'a' ] },
14 200 => {
15 a => [ 245 , 'a' ],
16 f => [ 245 , 'f' ],
17 },
18 205 => { a => [ 250 , 'a' ] },
19 210 => {
20 a => [ 260 , 'a' ],
21 c => [ 260 , 'b' ],
22 d => [ 260 , 'c' ],
23 },
24 215 => {
25 a => [ 300 , 'a' ],
26 c => [ 300 , 'b' ],
27 d => [ 300 , 'c' ],
28 },
29 700 => {
30 a => [ 100 , 'a' ],
31 },
32 };
33
34 our $mech = WWW::Mechanize->new();
35 our $hits;
36
37 sub diag {
38 print "# ", @_, $/;
39 }
40
41 # Koha Z39.50 query:
42 #
43 # Bib-1 @and @and @and @and @and @and @and @or
44 # @attr 1=8 isbn-issn
45 # @attr 1=7 isbn-issn
46 # @attr 1=4 title
47 # @attr 1=1003 author
48 # @attr 1=16 dewey
49 # @attr 1=21 subject-holding
50 # @attr 1=12 control-no
51 # @attr 1=1007 standard-id
52 # @attr 1=1016 any
53
54 our $usemap = {
55 8 => 'BN', # FIXME check
56 7 => 'SN', # FIXME check
57 4 => 'TI',
58 1003 => 'TI',
59 16 => 'CU',
60 21 => 'SU',
61 # 12 => '',
62 # 1007 => '',
63 # 1016 => '',
64
65 };
66
67 sub search {
68 my ( $self, $query ) = @_;
69
70 die "need query" unless defined $query;
71
72 my $url = 'http://cobiss.izum.si/scripts/cobiss?ukaz=GETID&lani=en';
73
74 diag "get $url";
75
76 $mech->get( $url );
77
78 diag "got session";
79
80 $mech->follow_link( text_regex => qr/union/ );
81
82 diag "switch to advanced form (select)";
83
84 $mech->follow_link( url_regex => qr/mode=3/ );
85
86 diag "submit search $query";
87
88 $mech->submit_form(
89 fields => {
90 'SS1' => $query,
91 },
92 );
93
94 $hits = 0;
95 if ( $mech->content =~ m{hits:\s*<b>\s*(\d+)\s*</b>}s ) {
96 $hits = $1;
97 } else {
98 diag "get't find results in ", $mech->content;
99 return;
100 }
101
102 diag "got $hits results, get first one";
103
104 $mech->follow_link( url_regex => qr/ukaz=DISP/ );
105
106 diag "in COMARC format";
107
108 $mech->follow_link( url_regex => qr/fmt=13/ );
109 }
110
111
112 sub fetch_rec {
113 my ($self,$format) = @_;
114
115 $format ||= 'unimarc';
116
117 die "unknown format: $format" unless $format =~ m{(uni|us)marc};
118
119 my $comarc;
120
121 if ( $mech->content =~ m{<pre>\s*(.+?(\d+)\.\s+ID=(\d+).+?)\s*</pre>}s ) {
122
123 my $comarc = $1;
124 my $nr = $2;
125 my $id = $3;
126
127 diag "fetch_marc $nr [$id] $format";
128
129 $comarc =~ s{</?b>}{}gs;
130 $comarc =~ s{<font[^>]*>}{<s>}gs;
131 $comarc =~ s{</font>}{<e>}gs;
132
133 open(my $out, '>:utf8', "comarc/$id");
134 print $out $comarc;
135 close($out);
136
137 print $comarc;
138
139 my $marc = MARC::Record->new;
140
141 foreach my $line ( split(/[\r\n]+/, $comarc) ) {
142
143 if ( $line !~ s{^(\d\d\d)([01 ])([01 ])}{} ) {
144 diag "SKIP: $line";
145 } else {
146 our @f = ( $1, $2, $3 );
147 $line .= "<eol>";
148
149 if ( $format eq 'unimarc' ) {
150
151 diag dump(@f), "line: $line";
152 sub sf_uni {
153 warn "sf ",dump(@_);
154 push @f, @_;
155 }
156 $line =~ s{<s>(\w)<e>([^<]+)\s*}{sf_uni($1, $2)}ges;
157 diag "f:", dump(@f), " left: |$line|";
158 $marc->add_fields( @f );
159
160 } elsif ( $format eq 'usmarc' ) {
161
162 my ( $f, $i1, $i2 ) = @f;
163
164 our $out = {};
165
166 sub sf_us {
167 my ($f,$sf,$v) = @_;
168 if ( my $m = $cobiss_marc21->{$f}->{$sf} ) {
169 push @{ $out->{ $m->[0] } }, ( $m->[1], $v );
170 }
171 return;
172 }
173 $line =~ s{<s>(\w)<e>([^<]+)\s*}{sf_us($f,$1, $2)}ges;
174
175 diag "converted marc21 ",dump( $out );
176
177 foreach my $f ( keys %$out ) {
178 $marc->add_fields( $f, $i1, $i2, @{ $out->{$f} } );
179 }
180 }
181 }
182 }
183
184 my $path = "marc/$id.$format";
185
186 open($out, '>:utf8', $path);
187 print $out $marc->as_usmarc;
188 close($out);
189
190 diag "created $path ", -s $path, " bytes";
191
192 diag $marc->as_formatted;
193
194 $nr++;
195 $mech->follow_link( url_regex => qr/rec=$nr/ );
196
197 return $marc->as_usmarc;
198 } else {
199 die "can't fetch COMARC format from ", $mech->content;
200 }
201
202 }
203
204 1;

  ViewVC Help
Powered by ViewVC 1.1.26