Contents of /trunk/spider/progspider

#!/usr/local/bin/perl -w
use strict;
use File::Find;


my $dir = shift @ARGV || die "usage: $0 [dir]";

my $basedir = $0;
$basedir =~ s,/[^/]+$,/,;
require "$basedir/filter.pm";

find({ wanted => \&file, 
        follow => 1,
        no_chdir => 1
}, $dir);

sub file {

        return if (! -f || ! m/\.html*/i);

        my $path = $_;

        open(F,"$path") || die "can't open file: $path";
#       print STDERR "$path\n";
        my $contents;
        while(<F>) {
                $contents .= $_;
        }

        $contents = filter($contents);

#       die "zero size content in '$path'" if (! $contents);
        return if (! $contents);        # don't die on empty files

        my $mtime = time;
        my $size = length $contents;

#       print STDERR " [$size]\n";

        # Output the document (to swish)
        print <<EOF;
Path-Name: $path
Content-Length: $size
Last-Mtime: $mtime
Document-Type: HTML

EOF

        print $contents;

}
1	#!/usr/local/bin/perl -w
2	use strict;
3	use File::Find;
4
5
6	my $dir = shift @ARGV \|\| die "usage: $0 [dir]";
7
8	my $basedir = $0;
9	$basedir =~ s,/[^/]+$,/,;
10	require "$basedir/filter.pm";
11
12	find({ wanted => \&file,
13	follow => 1,
14	no_chdir => 1
15	}, $dir);
16
17	sub file {
18
19	return if (! -f \|\| ! m/\.html*/i);
20
21	my $path = $_;
22
23	open(F,"$path") \|\| die "can't open file: $path";
24	# print STDERR "$path\n";
25	my $contents;
26	while(<F>) {
27	$contents .= $_;
28	}
29
30	$contents = filter($contents);
31
32	# die "zero size content in '$path'" if (! $contents);
33	return if (! $contents); # don't die on empty files
34
35	my $mtime = time;
36	my $size = length $contents;
37
38	# print STDERR " [$size]\n";
39
40	# Output the document (to swish)
41	print <<EOF;
42	Path-Name: $path
43	Content-Length: $size
44	Last-Mtime: $mtime
45	Document-Type: HTML
46
47	EOF
48
49	print $contents;
50
51	}