--- trunk/spider/progspider	2004/01/20 18:13:32	50
+++ trunk/spider/progspider	2004/01/23 13:10:40	56
@@ -1,14 +1,27 @@
 #!/usr/local/bin/perl -w
 use strict;
 use File::Find;
+use Getopt::Long;
 
+my $collection;		# name which will be inserted
+my $path_add;		# add additional info in path
+my $verbose;
+
+my $result = GetOptions(
+	"collection=s" => \$collection,
+	"path=s" => \$path_add,
+	"verbose!" => \$verbose,
+	"debug!" => \$verbose,
+);
 
 my $dir = shift @ARGV || die "usage: $0 [dir]";
 
+
 my $basedir = $0;
 $basedir =~ s,/[^/]+$,/,;
 require "$basedir/filter.pm";
 
+
 find({ wanted => \&file, 
 	follow => 1,
 	no_chdir => 1
@@ -18,24 +31,32 @@
 
 	return if (! -f || ! m/\.html*/i);
 
+	# skip index files
+	return if (m/index_[a-z]\.html*/i || m/index_symbol\.html*/i);
+
 	my $path = $_;
 
 	open(F,"$path") || die "can't open file: $path";
-#	print STDERR "$path\n";
+	print STDERR "$path" if ($verbose);
 	my $contents;
 	while(<F>) {
-		$contents .= $_;
+		$contents .= "$_";
 	}
+	$contents .= "\n\n";
 
-	$contents = filter($contents);
+	$contents = filter($contents,$collection);
 
 #	die "zero size content in '$path'" if (! $contents);
 	return if (! $contents);	# don't die on empty files
 
 	my $mtime = time;
+	use bytes;
 	my $size = length $contents;
 
-#	print STDERR " [$size]\n";
+	print STDERR " [$size]\n" if ($verbose);
+
+	# add optional components to path
+	$path .= " $path_add" if ($path_add);
 
 	# Output the document (to swish)
 	print <<EOF;
@@ -45,7 +66,6 @@
 Document-Type: HTML
 
 EOF
-
 	print $contents;
 
 }