Contents of /trunk/spider/progspider

#!/usr/local/bin/perl -w
use strict;
use File::Find;
use Getopt::Long;
use File::Which;

my $collection;         # name which will be inserted
my $path_add;           # add additional info in path
my $verbose;

#$verbose = 1;

my $result = GetOptions(
        "collection=s" => \$collection,
        "path=s" => \$path_add,
        "verbose!" => \$verbose,
        "debug!" => \$verbose,
);

my $dir = shift @ARGV || die "usage: $0 [dir]";

my $basedir = $0;
$basedir =~ s,/[^/]+$,/,;
require "$basedir/filter.pm";

my $pdftotext = which('pdftotext');

print STDERR "using $pdftotext to convert pdf into html\n" if ($pdftotext && $verbose);

find({ wanted => \&file, 
        follow => 1,
        no_chdir => 1
}, $dir);

sub file {

        my $path = $_;
        my $contents;

        if ($pdftotext && -f $path && $path =~ m/\.pdf$/i) {

                print STDERR "$path {converting}" if ($verbose);

                open(F,"$pdftotext -htmlmeta $path - |") || die "can't open $pdftotext with '$path'";
                my $html;
                while(<F>) {
                        # XXX why pdftotext barks if I try to use this is beyond me.
                        #$contents .= $_;

                        $html .= $_;
                }
                close(F);

                $contents = "<!-- html from $path -->\n$html\n";

        } else {

                return if (! -f $path || ! m/\.html*$/i);

                # skip index files
                return if (m/index_[a-z]\.html*/i || m/index_symbol\.html*/i);

                open(F,"$path") || die "can't open file: $path";
                print STDERR "$path" if ($verbose);
                while(<F>) {
                        $contents .= "$_";
                }
                $contents .= "\n\n";

                $contents = filter($contents,$collection);
        }

#       die "zero size content in '$path'" if (! $contents);
        return if (! $contents);        # don't die on empty files

        my $mtime = time;
        use bytes;
        my $size = length $contents;

        print STDERR " [$size]\n" if ($verbose);

        # add optional components to path
        $path .= " $path_add" if ($path_add);

        # Output the document (to swish)
        print <<EOF;
Path-Name: $path
Content-Length: $size
Last-Mtime: $mtime
Document-Type: HTML

EOF
        print $contents;

}
1	#!/usr/local/bin/perl -w
2	use strict;
3	use File::Find;
4	use Getopt::Long;
5	use File::Which;
6
7	my $collection; # name which will be inserted
8	my $path_add; # add additional info in path
9	my $verbose;
10
11	#$verbose = 1;
12
13	my $result = GetOptions(
14	"collection=s" => \$collection,
15	"path=s" => \$path_add,
16	"verbose!" => \$verbose,
17	"debug!" => \$verbose,
18	);
19
20	my $dir = shift @ARGV \|\| die "usage: $0 [dir]";
21
22	my $basedir = $0;
23	$basedir =~ s,/[^/]+$,/,;
24	require "$basedir/filter.pm";
25
26	my $pdftotext = which('pdftotext');
27
28	print STDERR "using $pdftotext to convert pdf into html\n" if ($pdftotext && $verbose);
29
30	find({ wanted => \&file,
31	follow => 1,
32	no_chdir => 1
33	}, $dir);
34
35	sub file {
36
37	my $path = $_;
38	my $contents;
39
40	if ($pdftotext && -f $path && $path =~ m/\.pdf$/i) {
41
42	print STDERR "$path {converting}" if ($verbose);
43
44	open(F,"$pdftotext -htmlmeta $path - \|") \|\| die "can't open $pdftotext with '$path'";
45	my $html;
46	while(<F>) {
47	# XXX why pdftotext barks if I try to use this is beyond me.
48	#$contents .= $_;
49
50	$html .= $_;
51	}
52	close(F);
53
54	$contents = "<!-- html from $path -->\n$html\n";
55
56	} else {
57
58	return if (! -f $path \|\| ! m/\.html*$/i);
59
60	# skip index files
61	return if (m/index_[a-z]\.html/i \|\| m/index_symbol\.html/i);
62
63	open(F,"$path") \|\| die "can't open file: $path";
64	print STDERR "$path" if ($verbose);
65	while(<F>) {
66	$contents .= "$_";
67	}
68	$contents .= "\n\n";
69
70	$contents = filter($contents,$collection);
71	}
72
73	# die "zero size content in '$path'" if (! $contents);
74	return if (! $contents); # don't die on empty files
75
76	my $mtime = time;
77	use bytes;
78	my $size = length $contents;
79
80	print STDERR " [$size]\n" if ($verbose);
81
82	# add optional components to path
83	$path .= " $path_add" if ($path_add);
84
85	# Output the document (to swish)
86	print <<EOF;
87	Path-Name: $path
88	Content-Length: $size
89	Last-Mtime: $mtime
90	Document-Type: HTML
91
92	EOF
93	print $contents;
94
95	}