--- trunk/spider/filter.pm	2004/01/17 23:57:55	46
+++ trunk/spider/filter.pm	2004/01/25 16:49:50	57
@@ -1,5 +1,8 @@
 sub filter {
 	my $contents = shift || return;
+
+#	my $verbose = 1;
+	
 	# if you don't want content to be indexed, include it in
 	# <noindex> foobar </noindex> tags or surround it with comments
 	# <!-- noindex --> foobar <!-- /noindex -->
@@ -8,7 +11,7 @@
 	$contents =~ s,<!--\s*noindex\s*-->.+?<!--\s*/noindex\s*-->,,isg;
 	$contents =~ s,<!--\s*noindex\s*-->.+?<!--\s*index\s*-->,,isg;
 	# this will remove all script from indexing content
-	$contents =~ s,<script>.+?</script>,,isg;
+	$contents =~ s,<script[^>]*>.+?</script>,,isg;
 	# remap Windows charset to ISO-8859-2
 	$contents =~ tr/氿炶鎶袔绕/桂捐姗挟绕/;	# 1250 -> iso8859-2
 	# this will fix badly formatted html in form:
@@ -36,6 +39,56 @@
 		$contents =~ s,(<title>)([^<]+)(</title>),$1$2: $title$3,gsi if ($title);
 
 	}
+
+	# is second argument collection?
+	my $collection = shift || return $contents;
+
+	# construct new title (from various parts of DocBook if available)
+	my $new_title;
+
+	if ($contents =~ m,<!--SafTocEntry="([^"]+)"-->,is) {
+		$new_title = $1;
+	} elsif ($contents =~ m,<h\d\sclass="docPartTitle"[^>]*>([^<]+)</h\d>,is) {
+		$new_title = $1;
+	} elsif ($contents =~ m,<h\d\sclass="docChapterTitle"[^>]*>([^<]+)</h\d>,is) {
+		$new_title = $1;
+	} elsif ($contents =~ m,<h\d\sclass="docSection1Title"[^>]*>([^<]+)</h\d>,is) {
+		$new_title = $1;
+	} else {
+		if ($contents =~ m,<title>([^<]+)</title>,is) {
+			$new_title = $1;
+		} elsif ($contents =~ m,<h\d[^>]*>([^<]+)</h\d>,is) {
+			$new_title = $1;
+		}
+	}
+
+	if ($new_title) {
+		# check if new title is same as collection name
+		my ($a,$b) = ($new_title,$collection);
+		$a =~ s/([^a-zA-Z])+/ /gs;
+		$b =~ s/([^a-zA-Z])+/ /gs;
+		if ($a =~ m/$b/i) {
+			$new_title = $collection;
+		} else {
+			$new_title = "$collection :: $new_title";
+		}
+	} else {
+		# fall-back to collection title
+		$new_title = $collection;
+	}
+
+	$new_title =~ s/\s\s+/ /g;
+
+	if ($contents =~ s,<title>(.*)</title>,<title>$new_title</title>,is) {
+		print STDERR "replace title '$1' with '$new_title'\n" if ($verbose);
+	} elsif (
+			# try to insert after <head>, <html> or at top
+		$contents =~ s,(<head>),$1<title>$new_title</title>,is ||
+		$contents =~ s,(<html>),$1<title>$new_title</title>,is ||
+			$contents =~ s,^,<title>$new_title</title>, ) {
+		print STDERR "adding new title '$new_title'\n" if ($verbose);
+	}
+
 	return $contents;
 }
 
