Annotation of /trunk/spider/progspider

#!/usr/local/bin/perl -w
use strict;
use File::Find;
use Getopt::Long;

my $collection;         # name which will be inserted
my $path_add;           # add additional info in path
my $verbose;

#$verbose = 1;

my $result = GetOptions(
        "collection=s" => \$collection,
        "path=s" => \$path_add,
        "verbose!" => \$verbose,
        "debug!" => \$verbose,
);

my $dir = shift @ARGV || die "usage: $0 [dir]";


my $basedir = $0;
$basedir =~ s,/[^/]+$,/,;
require "$basedir/filter.pm";


find({ wanted => \&file, 
        follow => 1,
        no_chdir => 1
}, $dir);

sub file {

        return if (! -f || ! m/\.html*/i);

        # skip index files
        return if (m/index_[a-z]\.html*/i || m/index_symbol\.html*/i);

        my $path = $_;

        open(F,"$path") || die "can't open file: $path";
        print STDERR "$path" if ($verbose);
        my $contents;
        while(<F>) {
                $contents .= "$_";
        }
        $contents .= "\n\n";

        $contents = filter($contents,$collection);

#       die "zero size content in '$path'" if (! $contents);
        return if (! $contents);        # don't die on empty files

        my $mtime = time;
        use bytes;
        my $size = length $contents;

        print STDERR " [$size]\n" if ($verbose);

        # add optional components to path
        $path .= " $path_add" if ($path_add);

        # Output the document (to swish)
        print <<EOF;
Path-Name: $path
Content-Length: $size
Last-Mtime: $mtime
Document-Type: HTML

EOF
        print $contents;

}
1	dpavlin	46	#!/usr/local/bin/perl -w
2			use strict;
3			use File::Find;
4	dpavlin	56	use Getopt::Long;
5	dpavlin	46
6	dpavlin	56	my $collection; # name which will be inserted
7			my $path_add; # add additional info in path
8			my $verbose;
9	dpavlin	46
10	dpavlin	57	#$verbose = 1;
11
12	dpavlin	56	my $result = GetOptions(
13			"collection=s" => \$collection,
14			"path=s" => \$path_add,
15			"verbose!" => \$verbose,
16			"debug!" => \$verbose,
17			);
18
19	dpavlin	46	my $dir = shift @ARGV \|\| die "usage: $0 [dir]";
20
21	dpavlin	56
22	dpavlin	46	my $basedir = $0;
23			$basedir =~ s,/[^/]+$,/,;
24			require "$basedir/filter.pm";
25
26	dpavlin	56
27	dpavlin	46	find({ wanted => \&file,
28			follow => 1,
29			no_chdir => 1
30			}, $dir);
31
32			sub file {
33
34			return if (! -f \|\| ! m/\.html*/i);
35
36	dpavlin	56	# skip index files
37			return if (m/index_[a-z]\.html/i \|\| m/index_symbol\.html/i);
38
39	dpavlin	46	my $path = $_;
40
41			open(F,"$path") \|\| die "can't open file: $path";
42	dpavlin	56	print STDERR "$path" if ($verbose);
43	dpavlin	46	my $contents;
44			while(<F>) {
45	dpavlin	56	$contents .= "$_";
46	dpavlin	46	}
47	dpavlin	56	$contents .= "\n\n";
48	dpavlin	46
49	dpavlin	56	$contents = filter($contents,$collection);
50	dpavlin	46
51	dpavlin	50	# die "zero size content in '$path'" if (! $contents);
52			return if (! $contents); # don't die on empty files
53
54	dpavlin	46	my $mtime = time;
55	dpavlin	56	use bytes;
56	dpavlin	46	my $size = length $contents;
57
58	dpavlin	56	print STDERR " [$size]\n" if ($verbose);
59	dpavlin	46
60	dpavlin	56	# add optional components to path
61			$path .= " $path_add" if ($path_add);
62
63	dpavlin	46	# Output the document (to swish)
64			print <<EOF;
65			Path-Name: $path
66			Content-Length: $size
67			Last-Mtime: $mtime
68			Document-Type: HTML
69
70			EOF
71			print $contents;
72
73			}