--- trunk/spider/swishspider 2003/03/24 09:57:44 30
+++ trunk/spider/swishspider 2003/06/01 11:45:19 40
@@ -18,7 +18,9 @@
my $no_parent_url;
if ($url =~ m/\s/) {
- ($url,$no_parent_url) = split(/\s/,$url,2);
+ ($no_parent_url,$url) = split(/\s/,$url,2);
+ # old scheme had URL, no parent and new is reverse
+ ($url,$no_parent_url) = ($no_parent_url,$url) if ($no_parent_url =~ m/$url/);
}
my $request = new HTTP::Request( "GET", $url );
@@ -39,7 +41,7 @@
if ($no_parent_url) {
if ($link =~ m/$no_parent_url/) {
# if this URL is below parent URL o.k....
- print RESP "$link $no_parent_url\n";
+ print RESP "$no_parent_url $link\n";
} else {
# if not, crawl just this page!
print RESP "$link $link\n";
@@ -59,20 +61,26 @@
my $contents = $response->content();
open( CONTENTS, ">$localpath.contents" ) || die( "Could not open contents file $localpath.contents\n" );
- # if you don't want content to be indexed, include it in
- # foobar tags or surround it with comments
- # foobar
- $contents =~ s,.+?,,isg;
- $contents =~ s,.+?,,isg;
- # this will remove all script from indexing content
- $contents =~ s,,,isg;
- # remap Windows charset to ISO-8859-2
- $contents =~ tr/šðžèæŠÐŽÈÆ/¹ð¾èæ©Ð®ÈÆ/; # 1250 -> iso8859-2
- # this will fix badly formatted html in form:
- #
some text
- # which will confuse indexer (or libxml2?)
- $contents =~ s/[\n\r]^(>)/$1\n/msg;
+ # fixup just HTML files
+ if ($response->header("content-type") =~ "text/html") {
+ # if you don't want content to be indexed, include it in
+ # foobar tags or surround it with comments
+ # foobar
+ $contents =~ s,.+?,,isg;
+ $contents =~ s,.+?,,isg;
+ # this will remove all script from indexing content
+ $contents =~ s,,,isg;
+ # remap Windows charset to ISO-8859-2
+ $contents =~ tr/šðžèæŠÐŽÈÆ/¹ð¾èæ©Ð®ÈÆ/; # 1250 -> iso8859-2
+ # this will fix badly formatted html in form:
+ # some text
+ # which will confuse indexer (or libxml2?)
+ $contents =~ s/[\n\r]^(>)/$1\n/msg;
+ # remove comments between and texi2html inserts them
+ # there and swish can't find document title then (libxml or swish bug?)
+ while ($contents =~ s/(.*)(.*)/$1$2/msi) { };
+ }
print CONTENTS $contents;
close( CONTENTS );
@@ -88,8 +96,8 @@
sub linkcb {
my($tag, %links) = @_;
- if (($tag eq "a" || $tag eq "area") && ($links{"href"})) {
- my $link = $links{"href"};
+ if (($tag eq "a" || $tag eq "area") && ($links{"href"}) || ($tag eq "frame" && $links{"src"})) {
+ my $link = $links{"href"} || $links{"src"};
#
# Remove fragments
@@ -113,7 +121,7 @@
if ($no_parent_url) {
if ($link =~ m/$no_parent_url/) {
- print LINKS "$link $no_parent_url\n";
+ print LINKS "$no_parent_url $link\n";
# print STDERR "using $link\n";
# } else {
# print STDERR "skipping $link\n";