Annotation of /trunk/spider/progspider

#!/usr/local/bin/perl -w
use strict;
use File::Find;


my $dir = shift @ARGV || die "usage: $0 [dir]";

my $basedir = $0;
$basedir =~ s,/[^/]+$,/,;
require "$basedir/filter.pm";

find({ wanted => \&file, 
        follow => 1,
        no_chdir => 1
}, $dir);

sub file {

        return if (! -f || ! m/\.html*/i);

        my $path = $_;

        open(F,"$path") || die "can't open file: $path";
#       print STDERR "$path\n";
        my $contents;
        while(<F>) {
                $contents .= $_;
        }

        $contents = filter($contents);

#       die "zero size content in '$path'" if (! $contents);
        return if (! $contents);        # don't die on empty files

        my $mtime = time;
        my $size = length $contents;

#       print STDERR " [$size]\n";

        # Output the document (to swish)
        print <<EOF;
Path-Name: $path
Content-Length: $size
Last-Mtime: $mtime
Document-Type: HTML

EOF

        print $contents;

}
1	dpavlin	46	#!/usr/local/bin/perl -w
2			use strict;
3			use File::Find;
4
5
6			my $dir = shift @ARGV \|\| die "usage: $0 [dir]";
7
8			my $basedir = $0;
9			$basedir =~ s,/[^/]+$,/,;
10			require "$basedir/filter.pm";
11
12			find({ wanted => \&file,
13			follow => 1,
14			no_chdir => 1
15			}, $dir);
16
17			sub file {
18
19			return if (! -f \|\| ! m/\.html*/i);
20
21			my $path = $_;
22
23			open(F,"$path") \|\| die "can't open file: $path";
24	dpavlin	50	# print STDERR "$path\n";
25	dpavlin	46	my $contents;
26			while(<F>) {
27			$contents .= $_;
28			}
29
30			$contents = filter($contents);
31
32	dpavlin	50	# die "zero size content in '$path'" if (! $contents);
33			return if (! $contents); # don't die on empty files
34
35	dpavlin	46	my $mtime = time;
36			my $size = length $contents;
37
38	dpavlin	48	# print STDERR " [$size]\n";
39	dpavlin	46
40			# Output the document (to swish)
41			print <<EOF;
42			Path-Name: $path
43			Content-Length: $size
44			Last-Mtime: $mtime
45			Document-Type: HTML
46
47			EOF
48
49			print $contents;
50
51			}