corp/esi/filter.pl

#!/usr/local/bin/perl -w

use LWP::UserAgent;
use strict;
use DBI;

my $debug=0;

my $dbh = DBI->connect("DBI:Pg:dbname=corp","","") || die $DBI::errstr;

my $ua = new LWP::UserAgent;
$ua->agent("pliva_harvester 0.0");
$ua->timeout(60);
$ua->env_proxy();

my $market=0;
my @markets = ('LSE','FTSE','bug1','bug2','bug3','bug4');
my @val;

my $close_time='21:21:21 CET';

sub insert {
        my $sql="insert into stocks values ('".shift(@val)."','".$markets[$market]."',".join(",",@val).")";
        $sql=~s/,\+(\d)/,$1/g;  # nuke + which pgsql doesn't like
        $dbh->do("$sql") || die "$sql\n".$DBI::errstr;
        print "$sql\n" if ($debug);
        $market++

}

my $req = HTTP::Request->new(GET =>"http://analysis.esi.co.uk/display.cgi?PLVD");
my $res = $ua->request($req);
if ($res->is_success) {
    foreach (split(/\n/, $res->content)) {
        chomp;
        if (m, at (\d+:\d\d) on (\d+\s\w+\s\d+),i) {
                my ($time,$date) = ($1,$2);
                insert() if (@val);
                undef @val;
                push @val,"$date $time";
                $dbh->do("delete from stocks where date='$date $close_time'");
#               print "delete from stocks where date='$date $close_time'";
        } elsif (m, at close on (\d+\s\w+\s\d+),i) {
                my ($time,$date) = ($close_time,$1);
                insert() if (@val);
                undef @val;
                push @val,"$date $time";
        }
        if (m,<td VALIGN="MIDDLE" ALIGN="CENTER" HEIGHT="30"><FONT SIZE="-1" FACE="Verdana"><b>([-\+]*\d+\.\d+)</b>,i) {
                push @val,$1;
        }

    }
    insert();
}

$dbh->disconnect;
1	dpavlin	1.1	#!/usr/local/bin/perl -w
2
3			use LWP::UserAgent;
4			use strict;
5			use DBI;
6
7	dpavlin	1.4	my $debug=0;
8
9	dpavlin	1.1	my $dbh = DBI->connect("DBI:Pg:dbname=corp","","") \|\| die $DBI::errstr;
10
11			my $ua = new LWP::UserAgent;
12			$ua->agent("pliva_harvester 0.0");
13			$ua->timeout(60);
14			$ua->env_proxy();
15
16			my $market=0;
17			my @markets = ('LSE','FTSE','bug1','bug2','bug3','bug4');
18			my @val;
19
20	dpavlin	1.3	my $close_time='21:21:21 CET';
21	dpavlin	1.1
22			sub insert {
23			my $sql="insert into stocks values ('".shift(@val)."','".$markets[$market]."',".join(",",@val).")";
24			$sql=~s/,\+(\d)/,$1/g; # nuke + which pgsql doesn't like
25	dpavlin	1.5	$dbh->do("$sql") \|\| die "$sql\n".$DBI::errstr;
26	dpavlin	1.4	print "$sql\n" if ($debug);
27	dpavlin	1.1	$market++
28
29			}
30
31			my $req = HTTP::Request->new(GET =>"http://analysis.esi.co.uk/display.cgi?PLVD");
32			my $res = $ua->request($req);
33			if ($res->is_success) {
34			foreach (split(/\n/, $res->content)) {
35			chomp;
36			if (m, at (\d+:\d\d) on (\d+\s\w+\s\d+),i) {
37			my ($time,$date) = ($1,$2);
38	dpavlin	1.2	insert() if (@val);
39	dpavlin	1.1	undef @val;
40	dpavlin	1.4	push @val,"$date $time";
41	dpavlin	1.1	$dbh->do("delete from stocks where date='$date $close_time'");
42			# print "delete from stocks where date='$date $close_time'";
43			} elsif (m, at close on (\d+\s\w+\s\d+),i) {
44			my ($time,$date) = ($close_time,$1);
45	dpavlin	1.2	insert() if (@val);
46	dpavlin	1.1	undef @val;
47			push @val,"$date $time";
48			}
49			if (m,<td VALIGN="MIDDLE" ALIGN="CENTER" HEIGHT="30"><FONT SIZE="-1" FACE="Verdana"><b>([-\+]*\d+\.\d+)</b>,i) {
50			push @val,$1;
51			}
52
53			}
54			insert();
55			}
56
57			$dbh->disconnect;