/[swish]/trunk/spider/progspider
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /trunk/spider/progspider

Parent Directory Parent Directory | Revision Log Revision Log


Revision 95 - (show annotations)
Sun Apr 24 16:33:53 2005 UTC (18 years, 10 months ago) by dpavlin
File size: 3122 byte(s)
added --exclude path

1 #!/usr/bin/perl -w
2 use strict;
3 use File::Find;
4 use Getopt::Long;
5 use File::Which;
6
7 my $collection; # name which will be inserted
8 my $path_add; # add additional info in path
9 my $verbose;
10 my $exclude;
11
12 #$verbose = 1;
13
14 my $result = GetOptions(
15 "collection=s" => \$collection,
16 "path=s" => \$path_add,
17 "verbose!" => \$verbose,
18 "debug!" => \$verbose,
19 "exclude=s" => \$exclude,
20 );
21
22 my $dir = shift @ARGV || die "usage: $0 [dir]";
23
24 my $basedir = $0;
25 $basedir =~ s,/[^/]+$,/,;
26 require "$basedir/filter.pm";
27
28 my $pdftotext = which('pdftotext');
29
30 select(STDERR); $|=1;
31 select(STDOUT); $|=1;
32
33 print STDERR "using $pdftotext to convert pdf into html\n" if ($pdftotext && $verbose);
34
35 find({ wanted => \&file,
36 follow => 1,
37 no_chdir => 1
38 }, $dir);
39
40 sub dump_contents($$$) {
41 my ($contents,$mtime,$path) = @_;
42
43 return unless ($contents); # don't die on empty files
44
45 if ($exclude && $path =~ m/$exclude/i) {
46 print STDERR "skip: $path\n" if ($verbose);
47 return;
48 }
49
50 use bytes;
51 my $size = length $contents;
52
53 print STDERR " [$size]" if ($verbose);
54
55 # Output the document (to swish)
56 print <<EOF;
57 Path-Name: $path
58 Content-Length: $size
59 Last-Mtime: $mtime
60 Document-Type: html*
61
62 EOF
63 print $contents;
64
65 }
66
67 sub file {
68
69 my $path = $_;
70 my $contents;
71
72 return if (-l $path);
73
74 if ($pdftotext && -f $path && $path =~ m/\.pdf$/i) {
75
76 print STDERR "$path {converting}" if ($verbose);
77
78 open(F,"$pdftotext -htmlmeta \"$path\" - |") || die "can't open $pdftotext with '$path'";
79 my $html;
80 while(<F>) {
81 # XXX why pdftotext barks if I try to use this is beyond me.
82 #$contents .= $_;
83
84 $html .= $_;
85 }
86 close(F);
87
88 return if (! $html);
89
90 my $file_only = $path;
91 $file_only =~ s/^.*\/([^\/]+)$/$1/g;
92
93 my ($pre_html,$pages,$post_html) = ('<html><head><title>$path :: page ##page_nr##</title></head><body><pre>',$html,'</pre></body></html>');
94
95 ($pre_html,$pages,$post_html) = ($1,$2,$3) if ($html =~ m/^(<html>.+?<pre>)(.+)(<\/pre>.+?)$/si);
96
97 if ($collection) {
98 $pre_html =~ s/<title>(.+?)<\/title>/<title>$collection :: page ##page_nr##<\/title>/si;
99 } else {
100 $pre_html =~ s/<title>(.+?)<\/title>/<title>$1 :: page ##page_nr##<\/title>/si ||
101 $pre_html =~ s/<title><\/title>/<title>$file_only :: page ##page_nr##<\/title>/si;
102 }
103
104 my $page_nr = 1;
105 foreach my $page (split(/\f/s,$pages)) {
106 print STDERR " $page_nr" if ($verbose);
107 my $pre_tmp = $pre_html;
108 $pre_tmp =~ s/##page_nr##/$page_nr<\/title>/s;
109 dump_contents($pre_tmp . $page . $post_html,time(), $path) if ($page !~ m/^\s*$/s);
110 $page_nr++;
111 }
112
113 } else {
114
115 return if (! -f $path || ! m/\.(html*|php|pl|txt|info|log|text)$/i);
116
117 # skip index files
118 return if (m/index_[a-z]\.html*/i || m/index_symbol\.html*/i);
119
120 open(F,"$path") || die "can't open file: $path";
121 print STDERR "$path" if ($verbose);
122 while(<F>) {
123 $contents .= "$_";
124 }
125 $contents .= "\n\n";
126
127 $contents = filter($contents,$collection);
128
129 # add optional components to path
130 $path .= " $path_add" if ($path_add);
131
132 dump_contents($contents,time(), $path);
133 }
134
135 print STDERR "\n" if ($verbose);
136 # die "zero size content in '$path'" if (! $contents);
137
138 }
139

Properties

Name Value
cvs2svn:cvs-rev 1.6
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26