viewvc.cgi/Z3950-HTML-Scraper/server.pl

#!/usr/bin/perl -w

use Net::Z3950::SimpleServer;
use Net::Z3950::OID;
use COBISS;
use strict;

my $max_records = 3; # XXX configure this
my $max_result_sets = 10;

sub diag {
        print "# ", @_, $/;
}

sub InitHandle {
    my $this    = shift;
    my $session = {};

    $this->{HANDLE}   = $session;
    $this->{IMP_NAME} = "Z39.50 HTML scraping bot";
    $this->{IMP_VER}  = "0.1";
    $session->{SETS}  = {};
}

use Data::Dumper;

sub SearchHandle {
    my $this    = shift;

diag "SearchHandle ",Dumper($this);

    my $session = $this->{HANDLE};
    my $rpn     = $this->{RPN};
    my $query;

    eval { $query = $rpn->{query}->render(); };
    if ( $@ && ref($@) ) {    ## Did someone/something report any errors?
        $this->{ERR_CODE} = $@->{errcode};
        $this->{ERR_STR}  = $@->{errstr};
        return;
    }

diag "search for $query";

    my $setname  = $this->{SETNAME};
    my $repl_set = $this->{REPL_SET};
    my $result;
    unless ( $result = COBISS->search( $query ) ) {
        $this->{ERR_CODE} = 108;
        return;
    }
    my $hits = $CROBISS::hits || diag "no results for $query";
    my $rs   = {
        lower => 1,
        upper => $hits < $max_records ? $max_records : $hits,
        data  => $result->{'resultElements'},   # FIXME
    };
    my $sets = $session->{SETS};

    if ( defined( $sets->{$setname} ) && !$repl_set ) {
        $this->{ERR_CODE} = 21;
        return;
    }
    if ( scalar keys %$sets >= $max_result_sets ) {
        $this->{ERR_CODE} = 112;
        $this->{ERR_STR}  = "Max number is $max_result_sets";
        return;
    }
    $sets->{$setname} = $rs;
    $this->{HITS} = $session->{HITS} = $hits;
    $session->{QUERY} = $query;
}

sub FetchHandle {
    my $this     = shift;
    my $session  = $this->{HANDLE};
    my $setname  = $this->{SETNAME};
    my $req_form = $this->{REQ_FORM};
    my $offset   = $this->{OFFSET};
    my $sets     = $session->{SETS};
    my $hits     = $session->{HITS};
    my $rs;
    my $record;

diag Dumper( $this );

    if ( !defined( $rs = $sets->{$setname} ) ) {
        $this->{ERR_CODE} = 30;
        return;
    }
    if ( $offset > $hits ) {
        $this->{ERR_CODE} = 13;
        return;
    }
    $this->{BASENAME} = "HtmlZ3950";

#    if ( !defined($req_form) || ( $req_form eq &Net::Z3950::OID::xml ) )
    if (0)
    {    ## XML records
        $this->{REP_FORM} = &Net::Z3950::OID::xml;
        $this->{RECORD}   = '<xml>FIXME: not implementd</xml>';
    }
    elsif ( $req_form eq &Net::Z3950::OID::unimarc ) {  # FIXME convert to usmarc
        $this->{REP_FORM} = &Net::Z3950::OID::unimarc;
        $this->{RECORD} = COBISS->fetch_marc;
    }
    else {    ## Unsupported record format
        $this->{ERR_CODE} = 239;
        $this->{ERR_STR}  = $req_form;
        return;
    }
    if ( $offset == $hits ) {
        $this->{LAST} = 1;
    }
    else {
        $this->{LAST} = 0;
    }
}

sub CloseHandle {
    my $this = shift;
}

my $z = new Net::Z3950::SimpleServer(
    INIT   => \&InitHandle,
    SEARCH => \&SearchHandle,
    FETCH  => \&FetchHandle,
    CLOSE  => \&CloseHandle
);
$z->launch_server( $0, @ARGV );

package Net::Z3950::RPN::And;

sub render {
    my $this = shift;
    return $this->[0]->render() . ' AND ' . $this->[1]->render();
}

package Net::Z3950::RPN::Or;

sub render {
    my $this = shift;
    return $this->[0]->render() . ' OR ' . $this->[1]->render();
}

package Net::Z3950::RPN::AndNot;

sub render {
    my $this = shift;
    return $this->[0]->render() . ' AND NOT ' . $this->[1]->render();
}

package Net::Z3950::RPN::Term;

use COBISS;

sub render {
    my $this       = shift;

print "render ", $this;

    my $attributes = {};
    my $prefix     = "";
    foreach my $attr ( @{ $this->{attributes} } ) {
        my $type  = $attr->{attributeType};
        my $value = $attr->{attributeValue};
        $attributes->{$type} = $value;
    }
    if ( defined( my $use = $attributes->{1} ) ) {
        if ( defined( my $field = COBISS::usemap($use) ) ) {
            $prefix = $field;
        }
        else {
            die { errcode => 114, errstr => $use }; ## Unsupported use attribute
        }
    }
    if ( defined( my $rel = $attributes->{2} ) )
    {    ## No relation attributes supported
        if ( $rel != 3 ) {
            die { errcode => 117, errstr => $rel };
        }
    }
    if ( defined( my $pos = $attributes->{3} ) )
    {    ## No position attributes either
        if ( $pos != 3 ) {
            die { errcode => 119, errstr => $pos };
        }
    }
    if ( defined( my $struc = $attributes->{4} ) ) {    ## No structure
        if ( ( $struc != 1 ) && ( $struc != 2 ) ) {
            die { errcode => 118, errstr => $struc };
        }
    }
    if ( defined( $attributes->{5} ) ) {                ## No truncation
        die { errcode => 113, errstr => 5 };
    }
    my $comp = $attributes->{6};
    if ($prefix) {
        if ( defined($comp) && ( $comp >= 2 ) ) {
            $prefix = "all$prefix= ";
        }
        else {
            $prefix = "$prefix=";
        }
    }

    my $q = $prefix . $this->{term};
        print "# q: $q\n";
        return $q;
}

1	dpavlin	6	#!/usr/bin/perl -w
2
3			use Net::Z3950::SimpleServer;
4			use Net::Z3950::OID;
5			use COBISS;
6			use strict;
7
8			my $max_records = 3; # XXX configure this
9			my $max_result_sets = 10;
10
11			sub diag {
12			print "# ", @_, $/;
13			}
14
15			sub InitHandle {
16			my $this = shift;
17			my $session = {};
18
19			$this->{HANDLE} = $session;
20			$this->{IMP_NAME} = "Z39.50 HTML scraping bot";
21			$this->{IMP_VER} = "0.1";
22			$session->{SETS} = {};
23			}
24
25			use Data::Dumper;
26
27			sub SearchHandle {
28			my $this = shift;
29
30			diag "SearchHandle ",Dumper($this);
31
32			my $session = $this->{HANDLE};
33			my $rpn = $this->{RPN};
34			my $query;
35
36			eval { $query = $rpn->{query}->render(); };
37			if ( $@ && ref($@) ) { ## Did someone/something report any errors?
38			$this->{ERR_CODE} = $@->{errcode};
39			$this->{ERR_STR} = $@->{errstr};
40			return;
41			}
42
43			diag "search for $query";
44
45			my $setname = $this->{SETNAME};
46			my $repl_set = $this->{REPL_SET};
47			my $result;
48			unless ( $result = COBISS->search( $query ) ) {
49			$this->{ERR_CODE} = 108;
50			return;
51			}
52			my $hits = $CROBISS::hits \|\| diag "no results for $query";
53			my $rs = {
54			lower => 1,
55			upper => $hits < $max_records ? $max_records : $hits,
56			data => $result->{'resultElements'}, # FIXME
57			};
58			my $sets = $session->{SETS};
59
60			if ( defined( $sets->{$setname} ) && !$repl_set ) {
61			$this->{ERR_CODE} = 21;
62			return;
63			}
64			if ( scalar keys %$sets >= $max_result_sets ) {
65			$this->{ERR_CODE} = 112;
66			$this->{ERR_STR} = "Max number is $max_result_sets";
67			return;
68			}
69			$sets->{$setname} = $rs;
70			$this->{HITS} = $session->{HITS} = $hits;
71			$session->{QUERY} = $query;
72			}
73
74			sub FetchHandle {
75			my $this = shift;
76			my $session = $this->{HANDLE};
77			my $setname = $this->{SETNAME};
78			my $req_form = $this->{REQ_FORM};
79			my $offset = $this->{OFFSET};
80			my $sets = $session->{SETS};
81			my $hits = $session->{HITS};
82			my $rs;
83			my $record;
84
85			diag Dumper( $this );
86
87			if ( !defined( $rs = $sets->{$setname} ) ) {
88			$this->{ERR_CODE} = 30;
89			return;
90			}
91			if ( $offset > $hits ) {
92			$this->{ERR_CODE} = 13;
93			return;
94			}
95			$this->{BASENAME} = "HtmlZ3950";
96
97			# if ( !defined($req_form) \|\| ( $req_form eq &Net::Z3950::OID::xml ) )
98			if (0)
99			{ ## XML records
100			$this->{REP_FORM} = &Net::Z3950::OID::xml;
101			$this->{RECORD} = '<xml>FIXME: not implementd</xml>';
102			}
103			elsif ( $req_form eq &Net::Z3950::OID::unimarc ) { # FIXME convert to usmarc
104			$this->{REP_FORM} = &Net::Z3950::OID::unimarc;
105			$this->{RECORD} = COBISS->fetch_marc;
106			}
107			else { ## Unsupported record format
108			$this->{ERR_CODE} = 239;
109			$this->{ERR_STR} = $req_form;
110			return;
111			}
112			if ( $offset == $hits ) {
113			$this->{LAST} = 1;
114			}
115			else {
116			$this->{LAST} = 0;
117			}
118			}
119
120			sub CloseHandle {
121			my $this = shift;
122			}
123
124			my $z = new Net::Z3950::SimpleServer(
125			INIT => \&InitHandle,
126			SEARCH => \&SearchHandle,
127			FETCH => \&FetchHandle,
128			CLOSE => \&CloseHandle
129			);
130			$z->launch_server( $0, @ARGV );
131
132			package Net::Z3950::RPN::And;
133
134			sub render {
135			my $this = shift;
136			return $this->[0]->render() . ' AND ' . $this->[1]->render();
137			}
138
139			package Net::Z3950::RPN::Or;
140
141			sub render {
142			my $this = shift;
143			return $this->[0]->render() . ' OR ' . $this->[1]->render();
144			}
145
146			package Net::Z3950::RPN::AndNot;
147
148			sub render {
149			my $this = shift;
150			return $this->[0]->render() . ' AND NOT ' . $this->[1]->render();
151			}
152
153			package Net::Z3950::RPN::Term;
154
155			use COBISS;
156
157			sub render {
158			my $this = shift;
159
160			print "render ", $this;
161
162			my $attributes = {};
163			my $prefix = "";
164			foreach my $attr ( @{ $this->{attributes} } ) {
165			my $type = $attr->{attributeType};
166			my $value = $attr->{attributeValue};
167			$attributes->{$type} = $value;
168			}
169			if ( defined( my $use = $attributes->{1} ) ) {
170			if ( defined( my $field = COBISS::usemap($use) ) ) {
171			$prefix = $field;
172			}
173			else {
174			die { errcode => 114, errstr => $use }; ## Unsupported use attribute
175			}
176			}
177			if ( defined( my $rel = $attributes->{2} ) )
178			{ ## No relation attributes supported
179			if ( $rel != 3 ) {
180			die { errcode => 117, errstr => $rel };
181			}
182			}
183			if ( defined( my $pos = $attributes->{3} ) )
184			{ ## No position attributes either
185			if ( $pos != 3 ) {
186			die { errcode => 119, errstr => $pos };
187			}
188			}
189			if ( defined( my $struc = $attributes->{4} ) ) { ## No structure
190			if ( ( $struc != 1 ) && ( $struc != 2 ) ) {
191			die { errcode => 118, errstr => $struc };
192			}
193			}
194			if ( defined( $attributes->{5} ) ) { ## No truncation
195			die { errcode => 113, errstr => 5 };
196			}
197			my $comp = $attributes->{6};
198			if ($prefix) {
199			if ( defined($comp) && ( $comp >= 2 ) ) {
200			$prefix = "all$prefix= ";
201			}
202			else {
203			$prefix = "$prefix=";
204			}
205			}
206
207			my $q = $prefix . $this->{term};
208			print "# q: $q\n";
209			return $q;
210			}
211