/[swish]/trunk/spider/progspider
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /trunk/spider/progspider

Parent Directory Parent Directory | Revision Log Revision Log


Revision 66 - (show annotations)
Wed Mar 17 12:19:42 2004 UTC (16 years, 7 months ago) by dpavlin
File size: 2556 byte(s)
index pdf files page-by-page

1 #!/usr/local/bin/perl -w
2 use strict;
3 use File::Find;
4 use Getopt::Long;
5 use File::Which;
6
7 my $collection; # name which will be inserted
8 my $path_add; # add additional info in path
9 my $verbose;
10
11 #$verbose = 1;
12
13 my $result = GetOptions(
14 "collection=s" => \$collection,
15 "path=s" => \$path_add,
16 "verbose!" => \$verbose,
17 "debug!" => \$verbose,
18 );
19
20 my $dir = shift @ARGV || die "usage: $0 [dir]";
21
22 my $basedir = $0;
23 $basedir =~ s,/[^/]+$,/,;
24 require "$basedir/filter.pm";
25
26 my $pdftotext = which('pdftotext');
27
28 select(STDERR); $|=1;
29 select(STDOUT); $|=1;
30
31 print STDERR "using $pdftotext to convert pdf into html\n" if ($pdftotext && $verbose);
32
33 find({ wanted => \&file,
34 follow => 1,
35 no_chdir => 1
36 }, $dir);
37
38 sub dump_contents($$$) {
39 my ($contents,$mtime,$path) = @_;
40
41 return if (! $contents); # don't die on empty files
42
43 use bytes;
44 my $size = length $contents;
45
46 print STDERR " [$size]" if ($verbose);
47
48 # Output the document (to swish)
49 print <<EOF;
50 Path-Name: $path
51 Content-Length: $size
52 Last-Mtime: $mtime
53 Document-Type: HTML
54
55 EOF
56 print $contents;
57
58 }
59
60 sub file {
61
62 my $path = $_;
63 my $contents;
64
65 if ($pdftotext && -f $path && $path =~ m/\.pdf$/i) {
66
67 print STDERR "$path {converting}" if ($verbose);
68
69 open(F,"$pdftotext -htmlmeta \"$path\" - |") || die "can't open $pdftotext with '$path'";
70 my $html;
71 while(<F>) {
72 # XXX why pdftotext barks if I try to use this is beyond me.
73 #$contents .= $_;
74
75 $html .= $_;
76 }
77 close(F);
78
79 my ($pre_html,$pages,$post_html) = ('<html><head><title>$path :: page ##page_nr##</title></head><body><pre>',$html,'</pre></body></html>');
80
81 ($pre_html,$pages,$post_html) = ($1,$2,$3) if ($html =~ m/^(<html>.+<pre>)(.+)(<\/pre>.+)$/si);
82
83 $pre_html =~ s/<title>(.+?)<\/title>/<title>$1 :: page ##page_nr##<\/title>/si;
84
85 my $page_nr = 1;
86 foreach my $page (split(/\f/,$pages)) {
87 my $pre_tmp = $pre_html;
88 $pre_tmp =~ s/##page_nr##/$page_nr<\/title>/s;
89 dump_contents($pre_tmp . $page . $post_html,time(), $path);
90 $page_nr++;
91 }
92
93 } else {
94
95 return if (! -f $path || ! m/\.html*$/i);
96
97 # skip index files
98 return if (m/index_[a-z]\.html*/i || m/index_symbol\.html*/i);
99
100 open(F,"$path") || die "can't open file: $path";
101 print STDERR "$path" if ($verbose);
102 while(<F>) {
103 $contents .= "$_";
104 }
105 $contents .= "\n\n";
106
107 $contents = filter($contents,$collection);
108
109 # add optional components to path
110 $path .= " $path_add" if ($path_add);
111
112 dump_contents($contents,time(), $path);
113 }
114
115 print STDERR "\n" if ($verbose);
116 # die "zero size content in '$path'" if (! $contents);
117
118 }
119

Properties

Name Value
cvs2svn:cvs-rev 1.6
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26