--- lib/Grep/Source.pm	2007/02/23 20:47:08	85
+++ lib/Grep/Source.pm	2007/02/24 12:32:31	100
@@ -7,13 +7,14 @@
 
 use Carp qw/verbose/;
 use Module::Pluggable search_path => 'Grep::Source', sub_name => 'sources', require => 1;
-use base qw(Class::Accessor);
+use base qw(Class::Accessor Jifty::Object);
 Grep::Source->mk_accessors( qw(feed uri q new_items collection) );
 
 use HTML::TreeBuilder;
 use WWW::Mechanize;
 use XML::Feed;
 use URI;
+use HTML::ResolveLink;
 
 use Data::Dump qw/dump/;
 
@@ -100,14 +101,16 @@
 
 	$self->uri( $uri );
 
-	Jifty->log->info( $message );
+	$self->log->info( $message );
 
 	$self->collection( Grep::Model::ItemCollection->new() );
 
 	my $class = $self->feed->source || 'Grep::Source::Feed';
-	Jifty->log->debug("using $class");
+	$self->log->debug("using $class");
 
-	$class->fetch( $self );
+	my $parent = $self;
+	$class->fetch( $parent );
+	undef $parent;
 
 	Grep::Search->finish if $self->new_items;
 
@@ -130,12 +133,18 @@
 
 	my $i = Grep::Model::Item->new();
 
-	my ($ok,$msg) = $i->load_or_create( @_ );
+	my $rec = {@_};
+
+	warn "resolving links";
+	my $resolver = HTML::ResolveLink->new( base => $rec->{link} );
+	$rec->{content} = $resolver->resolve( $rec->{content} );
+
+	my ($ok,$msg) = $i->load_or_create( %$rec );
 
 	$msg ||= '';
 
 	if ( $ok ) {
-		Jifty->log->debug("item ", $i->id, ": $msg");
+		$self->log->debug("item ", $i->id, ": $msg");
 		$self->collection->add_record( $i );
 
 		# is new record?
@@ -162,14 +171,14 @@
 	my $content = shift or die "no content?";
 
 	foreach my $s ( $self->sources ) {
-		Jifty->log->debug("testing source class $s");
+		$self->log->debug("testing source class $s");
 		if ( $s->can('content_have') ) {
 			my $regex =	$s->content_have( $content ) or
 				die "${s}->content_have didn't return anything";
 			die "${s}->content_have didn't return regex but ", dump( $regex ), " ref ", ref( $regex )
 				unless ( ref($regex) eq 'Regexp' );
 			if ( $content =~ $regex ) {
-				Jifty->log->debug("${s}->content_have succesful");
+				$self->log->debug("${s}->content_have succesful");
 				return $s;
 			}
 		}
@@ -194,25 +203,25 @@
 	die "no uri" unless ($uri);
 	die "feed is not a Grep::Model::Feed but ", ref $feed unless $feed->isa('Grep::Model::Feed');
 
-	sub save_html {
-		my ( $file, $content ) = @_;
-		if ( -w '/tmp/grep' ) {
-			open(my $f, '>', "/tmp/grep/${file}.html") or die "can't open $file: $!";
-			print $f $content or die "can't write to $file: $!";
-			close $f or die "can't close $file: $!";
-		}
+	sub mech_warn {
+		my $m = shift || return;
+		warn $m;
 	}
 
-	my $mech = WWW::Mechanize->new();
+	my $mech = WWW::Mechanize->new(
+		cookie_jar => {},
+		onwarn => \&mech_warn,
+		onerror => \&mech_warn,
+	);
 
 	$mech->get( $uri );
 
-	save_html( 'get', $mech->content );
+	$self->save( 'get.html', $mech->content );
 
-	if ( $args->{submit_form} ) {
-		warn "submit form on $uri\n";
-		$mech->submit_form( %{ $args->{submit_form} } ) or die "can't submit form";
-		save_html( 'submit', $mech->content );
+	if ( my $form = $args->{submit_form} ) {
+		warn "submit form on $uri with ", dump( $form ),"\n";
+		$mech->submit_form( %$form ) or die "can't submit form ", dump( $form );
+		$self->save( 'submit.html', $mech->content );
 	}
 
 	warn "parse result page\n";
@@ -230,7 +239,10 @@
 			( $_[0]->attr( $attr ) || '' ) eq $value;
 	});
 
-	die "can't find results wrapper <$el $attr=\"$value\">" unless ( $div );
+	if ( ! $div ) {
+		warn "can't find results wrapper <$el $attr=\"$value\">";
+		return;
+	}
 
 	my $max = 5;
 	my $nr = 1;
@@ -250,13 +262,13 @@
 			warn "fetching page: ",$a->as_text," from $page_uri\n";
 			if ( $mech->follow_link( url => $a->attr('href') ) ) {
 
-				save_html( "page-${nr}", $mech->content );
+				$self->save( "page-${nr}.html", $mech->content );
 
 				my $page_tree = HTML::TreeBuilder->new or die "can't create page tree";
 				$page_tree->parse( $mech->content ) or die "can't parse page at $page_uri";
 
-				my ( $el,$attr,$value ) = @{ $args->{scrape} };
-				my $div = $page_tree->look_down( '_tag', $el, sub { ( $_[0]->attr( $attr ) || '' ) eq $value } );
+				( $el,$attr,$value ) = @{ $args->{scrape} };
+				$div = $page_tree->look_down( '_tag', $el, sub { ( $_[0]->attr( $attr ) || '' ) eq $value } );
 
 				die "can't find <$el $attr=\"$value\">" unless ($div);
 
@@ -288,4 +300,24 @@
 
 }
 
+=head2 save
+
+  save( 'name', $content );
+
+Save dumps into C</tmp/grep> if writable
+
+=cut
+
+sub save {
+	my $self = shift;
+	my ( $file, $content ) = @_;
+	return unless ( defined($file) && defined($content) );
+	if ( -w '/tmp/grep' ) {
+		open(my $f, '>', "/tmp/grep/$file") or die "can't open $file: $!";
+		print $f $content or die "can't write to $file: $!";
+		close $f or die "can't close $file: $!";
+		$self->log->debug("saved $file ",length($content)," bytes");
+	}
+}
+
 1;