--- trunk/spider/swishspider	2002/06/04 06:39:53	1
+++ trunk/spider/swishspider	2003/03/24 09:57:44	30
@@ -59,10 +59,20 @@
     my $contents = $response->content();
 
     open( CONTENTS, ">$localpath.contents" ) || die( "Could not open contents file $localpath.contents\n" );
+    # if you don't want content to be indexed, include it in
+    # <noindex> foobar </noindex> tags or surround it with comments
+    # <!-- noindex --> foobar <!-- /noindex -->
     $contents =~ s,<noindex>.+?</noindex>,,isg;
     $contents =~ s,<!--\s*noindex\s*-->.+?<!--\s*/noindex\s*-->,,isg;
+    # this will remove all script from indexing content
     $contents =~ s,<script>.+?</script>,,isg;
+    # remap Windows charset to ISO-8859-2
     $contents =~ tr/氿炶鎶袔绕/桂捐姗挟绕/;	# 1250 -> iso8859-2
+    # this will fix badly formatted html in form:
+    # <head><title>some text</title
+    # ></head>
+    # which will confuse indexer (or libxml2?)
+    $contents =~ s/[\n\r]^(>)/$1\n/msg;
     print CONTENTS $contents;
     close( CONTENTS );
 
@@ -78,7 +88,7 @@
 
 sub linkcb {
     my($tag, %links) = @_;
-    if (($tag eq "a") && ($links{"href"})) {
+    if (($tag eq "a" || $tag eq "area") && ($links{"href"})) {
 	my $link = $links{"href"};
 
 	#
@@ -98,6 +108,9 @@
 	# hack for Apache directory listings
 	$link =~ s,/\?[NMSD]=[AD]$,/,g;
 
+	# speedup, skip pictures
+	return if ($link =~ m/\.(gif|jpg|png)/);
+
 	if ($no_parent_url) {
 			if ($link =~ m/$no_parent_url/) {
 				print LINKS "$link $no_parent_url\n";
