--- trunk/spider/swishspider 2003/03/24 09:57:44 30 +++ trunk/spider/swishspider 2003/06/01 11:45:19 40 @@ -18,7 +18,9 @@ my $no_parent_url; if ($url =~ m/\s/) { - ($url,$no_parent_url) = split(/\s/,$url,2); + ($no_parent_url,$url) = split(/\s/,$url,2); + # old scheme had URL, no parent and new is reverse + ($url,$no_parent_url) = ($no_parent_url,$url) if ($no_parent_url =~ m/$url/); } my $request = new HTTP::Request( "GET", $url ); @@ -39,7 +41,7 @@ if ($no_parent_url) { if ($link =~ m/$no_parent_url/) { # if this URL is below parent URL o.k.... - print RESP "$link $no_parent_url\n"; + print RESP "$no_parent_url $link\n"; } else { # if not, crawl just this page! print RESP "$link $link\n"; @@ -59,20 +61,26 @@ my $contents = $response->content(); open( CONTENTS, ">$localpath.contents" ) || die( "Could not open contents file $localpath.contents\n" ); - # if you don't want content to be indexed, include it in - # foobar tags or surround it with comments - # foobar - $contents =~ s,.+?,,isg; - $contents =~ s,.+?,,isg; - # this will remove all script from indexing content - $contents =~ s,,,isg; - # remap Windows charset to ISO-8859-2 - $contents =~ tr/Ў/Ю/; # 1250 -> iso8859-2 - # this will fix badly formatted html in form: - # some text - # which will confuse indexer (or libxml2?) - $contents =~ s/[\n\r]^(>)/$1\n/msg; + # fixup just HTML files + if ($response->header("content-type") =~ "text/html") { + # if you don't want content to be indexed, include it in + # foobar tags or surround it with comments + # foobar + $contents =~ s,.+?,,isg; + $contents =~ s,.+?,,isg; + # this will remove all script from indexing content + $contents =~ s,,,isg; + # remap Windows charset to ISO-8859-2 + $contents =~ tr/Ў/Ю/; # 1250 -> iso8859-2 + # this will fix badly formatted html in form: + # some text + # which will confuse indexer (or libxml2?) + $contents =~ s/[\n\r]^(>)/$1\n/msg; + # remove comments between and texi2html inserts them + # there and swish can't find document title then (libxml or swish bug?) + while ($contents =~ s/(.*)(.*)/$1$2/msi) { }; + } print CONTENTS $contents; close( CONTENTS ); @@ -88,8 +96,8 @@ sub linkcb { my($tag, %links) = @_; - if (($tag eq "a" || $tag eq "area") && ($links{"href"})) { - my $link = $links{"href"}; + if (($tag eq "a" || $tag eq "area") && ($links{"href"}) || ($tag eq "frame" && $links{"src"})) { + my $link = $links{"href"} || $links{"src"}; # # Remove fragments @@ -113,7 +121,7 @@ if ($no_parent_url) { if ($link =~ m/$no_parent_url/) { - print LINKS "$link $no_parent_url\n"; + print LINKS "$no_parent_url $link\n"; # print STDERR "using $link\n"; # } else { # print STDERR "skipping $link\n";