corp/esi/filter.pl

#!/usr/local/bin/perl -w

use LWP::UserAgent;
use strict;
use DBI;

my $debug=0;

$debug++ if (lc($ARGV[0]) eq "-d");

sub print_debug {
        return if (! $debug);
        open(DEBUG,">> debug") || warn "can't open debug file!";
        print DEBUG "###",@_;
        print @_;
        close(DEBUG);
}

print_debug("debug level $debug");

my $dbh = DBI->connect("DBI:Pg:dbname=corp","","") || die $DBI::errstr;

my $ua = new LWP::UserAgent;
$ua->agent("pliva_harvester 0.0");
$ua->timeout(60);
$ua->env_proxy();
$ua->proxy(['http', 'ftp'], 'http://proxy.pliva.hr:8080/');

my $market=0;
my @markets = ('LSE','FTSE','bug1','bug2','bug3','bug4');
my @val;

my $close_time='21:21:21 CET';

sub insert {
        my $sql="insert into stocks values ('".shift(@val)."','".$markets[$market]."',".join(",",@val).")";
        $sql=~s/,\+(\d)/,$1/g;  # nuke + which pgsql doesn't like
        $dbh->do("$sql") || die "$sql\n".$DBI::errstr;
        print_debug("sql: $sql\n");
        $market++;
}

my $req = HTTP::Request->new(GET =>"http://analysis.esi.co.uk/display.cgi?PLVD");
my $res = $ua->request($req);
if ($res->is_success) {
    print_debug("html: ".$res->content."\n");
    foreach (split(/[\n\r]+/, $res->content)) {
        chomp;
        print_debug("line: $_\n");
        if (m, at (\d+:\d\d) on (\d+\s\w+\s\d+),i) {
                my ($time,$date) = ($1,$2);
                print_debug("time: $time date: $date");
                insert() if (@val);
                undef @val;
                push @val,"$date $time";
                $dbh->do("delete from stocks where date='$date $close_time'");
#               print "delete from stocks where date='$date $close_time'";
        } elsif (m, at close on (\d+\s\w+\s\d+),i) {
                my ($time,$date) = ($close_time,$1);
                insert() if (@val);
                undef @val;
                push @val,"$date $time";
        }
        if (m,<td VALIGN="MIDDLE" ALIGN="CENTER" HEIGHT="30"><FONT SIZE="-1" FACE="Verdana"><b>([-\+]*\d+\.\d+)</b>,i) {
                push @val,$1;
                print_debug("val: $1");
        }

    }
    insert();
} else {
    warn "can't fetch stock data";
}

$dbh->disconnect;
1	dpavlin	1.1	#!/usr/local/bin/perl -w
2
3			use LWP::UserAgent;
4			use strict;
5			use DBI;
6
7	dpavlin	1.4	my $debug=0;
8
9	dpavlin	1.6	$debug++ if (lc($ARGV[0]) eq "-d");
10
11			sub print_debug {
12			return if (! $debug);
13			open(DEBUG,">> debug") \|\| warn "can't open debug file!";
14			print DEBUG "###",@_;
15			print @_;
16			close(DEBUG);
17			}
18
19			print_debug("debug level $debug");
20
21	dpavlin	1.1	my $dbh = DBI->connect("DBI:Pg:dbname=corp","","") \|\| die $DBI::errstr;
22
23			my $ua = new LWP::UserAgent;
24			$ua->agent("pliva_harvester 0.0");
25			$ua->timeout(60);
26			$ua->env_proxy();
27	dpavlin	1.6	$ua->proxy(['http', 'ftp'], 'http://proxy.pliva.hr:8080/');
28	dpavlin	1.1
29			my $market=0;
30			my @markets = ('LSE','FTSE','bug1','bug2','bug3','bug4');
31			my @val;
32
33	dpavlin	1.3	my $close_time='21:21:21 CET';
34	dpavlin	1.1
35			sub insert {
36			my $sql="insert into stocks values ('".shift(@val)."','".$markets[$market]."',".join(",",@val).")";
37			$sql=~s/,\+(\d)/,$1/g; # nuke + which pgsql doesn't like
38	dpavlin	1.5	$dbh->do("$sql") \|\| die "$sql\n".$DBI::errstr;
39	dpavlin	1.6	print_debug("sql: $sql\n");
40			$market++;
41	dpavlin	1.1	}
42
43			my $req = HTTP::Request->new(GET =>"http://analysis.esi.co.uk/display.cgi?PLVD");
44			my $res = $ua->request($req);
45			if ($res->is_success) {
46	dpavlin	1.6	print_debug("html: ".$res->content."\n");
47			foreach (split(/[\n\r]+/, $res->content)) {
48	dpavlin	1.1	chomp;
49	dpavlin	1.6	print_debug("line: $_\n");
50	dpavlin	1.1	if (m, at (\d+:\d\d) on (\d+\s\w+\s\d+),i) {
51			my ($time,$date) = ($1,$2);
52	dpavlin	1.6	print_debug("time: $time date: $date");
53	dpavlin	1.2	insert() if (@val);
54	dpavlin	1.1	undef @val;
55	dpavlin	1.4	push @val,"$date $time";
56	dpavlin	1.1	$dbh->do("delete from stocks where date='$date $close_time'");
57			# print "delete from stocks where date='$date $close_time'";
58			} elsif (m, at close on (\d+\s\w+\s\d+),i) {
59			my ($time,$date) = ($close_time,$1);
60	dpavlin	1.2	insert() if (@val);
61	dpavlin	1.1	undef @val;
62			push @val,"$date $time";
63			}
64			if (m,<td VALIGN="MIDDLE" ALIGN="CENTER" HEIGHT="30"><FONT SIZE="-1" FACE="Verdana"><b>([-\+]*\d+\.\d+)</b>,i) {
65			push @val,$1;
66	dpavlin	1.6	print_debug("val: $1");
67	dpavlin	1.1	}
68
69			}
70			insert();
71	dpavlin	1.6	} else {
72			warn "can't fetch stock data";
73	dpavlin	1.1	}
74
75			$dbh->disconnect;