--- lib/Grep/Source.pm	2007/03/25 11:41:54	113
+++ lib/Grep/Source.pm	2007/04/28 13:08:50	121
@@ -8,7 +8,7 @@
 use Carp qw/verbose/;
 use Module::Pluggable search_path => 'Grep::Source', sub_name => 'sources', require => 1;
 use base qw(Class::Accessor Jifty::Object);
-Grep::Source->mk_accessors( qw(feed uri q new_items collection search_obj) );
+Grep::Source->mk_accessors( qw(feed uri q new_items collection search_obj tree) );
 
 use HTML::TreeBuilder;
 use WWW::Mechanize;
@@ -195,6 +195,49 @@
 
 =cut
 
+sub element_by_triplet {
+	my $self = shift;
+
+	my $args = {@_};
+
+	my $tree = $args->{tree} || die "no tree";
+	my $message = $args->{message};
+	my $fatal = $args->{fatal};
+	die "no templates" unless defined( $args->{templates} );
+	my @templates = @{ $args->{templates} };
+
+	push @templates, ( undef, undef ) if ( $#templates == 0 );
+
+	die "wrapper doesn't have 3 elements but ", $#templates unless (
+		( $#templates + 1 ) % 3 == 0
+	);
+
+	my ( $result, $el, $attr, $value );
+
+	my @tags;
+
+	while ( @templates ) {
+		( $el,$attr,$value ) = splice( @templates, 0, 3 );
+		my $tag = $attr ? "<$el $attr=\"$value\">" : "<$el>";
+		push @tags, $tag;
+		$self->log->debug("looking for $message $tag");
+		$result = $tree->look_down( '_tag', $el, sub {
+				return 1 unless ( $attr && $value );
+				( $_[0]->attr( $attr ) || '' ) eq $value;
+		});
+		last if $result;
+	}
+
+	if ( ! $result ) {
+		my $msg = "can't find $message ", join(" ", @tags);
+		die $msg if ( $fatal );
+		warn $msg;
+		return;
+	}
+
+	return $result;
+}
+
 sub scrape {
 	my $self = shift;
 
@@ -232,19 +275,12 @@
 	my $tree = HTML::TreeBuilder->new or die "can't create html tree";
 	$tree->parse( $mech->content ) or die "can't parse fetched content";
 
-	die "wrapper doesn't have 3 elements but ", $#{ $args->{wrapper} } unless ( $#{ $args->{wrapper} } == 2 );
-	my ( $el,$attr,$value ) = @{ $args->{wrapper} };
-
-	$self->log->debug("looking for <$el $attr=\"$value\">");
-
-	my $div = $tree->look_down( '_tag', $el, sub {
-			( $_[0]->attr( $attr ) || '' ) eq $value;
-	});
-
-	if ( ! $div ) {
-		warn "can't find results wrapper <$el $attr=\"$value\">";
-		return;
-	}
+	my $div = $self->element_by_triplet(
+		tree => $tree,
+		templates => $args->{wrapper},
+		message => 'wrapper for all results',
+		fatal => 1
+	);
 
 	my $max = 15;
 	my $nr = 1;
@@ -252,8 +288,11 @@
 	my $base_uri = $uri;
 	$base_uri =~ s!\?.*$!!;
 
-	my @r = $div->look_down( '_tag', $args->{results} );
-	$self->log->debug("found ", scalar @r, " <$args->{results}>");
+	my @r = $self->element_by_triplet(
+		tree => $tree,
+		templates => $args->{results},
+		message => 'result element',
+	);
 
 	foreach my $dt ( @r ) {
 		my $a = $dt->look_down( '_tag', 'a', sub { $_[0]->attr('href') } );
@@ -271,11 +310,11 @@
 
 				my $page_tree = HTML::TreeBuilder->new or die "can't create page tree";
 				$page_tree->parse( $mech->content ) or die "can't parse page at $page_uri";
-
-				( $el,$attr,$value ) = @{ $args->{scrape} };
-				$div = $page_tree->look_down( '_tag', $el, sub { ( $_[0]->attr( $attr ) || '' ) eq $value } );
-
-				die "can't find <$el $attr=\"$value\">" unless ($div);
+				$div = $self->element_by_triplet(
+					tree => $page_tree,
+					message => "result $nr",
+					templates => $args->{scrape}
+				);
 
 				$self->add_record(
 					in_feed => $feed,
@@ -287,7 +326,7 @@
 #					author => 
 #					issued => 
 #					modified => 
-				);
+				) if ( $div );
 
 				$mech->back;
 				$page_tree->delete;
@@ -295,6 +334,8 @@
 			} else {
 				warn "can't follow uri $page_uri: $!\n";
 			}
+		} else {
+			$self->log->debug("result $nr doesn't have link inside, ignoring...");
 		}
 
 		last if ($nr == $max);