/[swish]/trunk/spider/progspider
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /trunk/spider/progspider

Parent Directory Parent Directory | Revision Log Revision Log


Revision 57 - (show annotations)
Sun Jan 25 16:49:50 2004 UTC (20 years, 2 months ago) by dpavlin
File size: 1323 byte(s)
various fixes

1 #!/usr/local/bin/perl -w
2 use strict;
3 use File::Find;
4 use Getopt::Long;
5
6 my $collection; # name which will be inserted
7 my $path_add; # add additional info in path
8 my $verbose;
9
10 #$verbose = 1;
11
12 my $result = GetOptions(
13 "collection=s" => \$collection,
14 "path=s" => \$path_add,
15 "verbose!" => \$verbose,
16 "debug!" => \$verbose,
17 );
18
19 my $dir = shift @ARGV || die "usage: $0 [dir]";
20
21
22 my $basedir = $0;
23 $basedir =~ s,/[^/]+$,/,;
24 require "$basedir/filter.pm";
25
26
27 find({ wanted => \&file,
28 follow => 1,
29 no_chdir => 1
30 }, $dir);
31
32 sub file {
33
34 return if (! -f || ! m/\.html*/i);
35
36 # skip index files
37 return if (m/index_[a-z]\.html*/i || m/index_symbol\.html*/i);
38
39 my $path = $_;
40
41 open(F,"$path") || die "can't open file: $path";
42 print STDERR "$path" if ($verbose);
43 my $contents;
44 while(<F>) {
45 $contents .= "$_";
46 }
47 $contents .= "\n\n";
48
49 $contents = filter($contents,$collection);
50
51 # die "zero size content in '$path'" if (! $contents);
52 return if (! $contents); # don't die on empty files
53
54 my $mtime = time;
55 use bytes;
56 my $size = length $contents;
57
58 print STDERR " [$size]\n" if ($verbose);
59
60 # add optional components to path
61 $path .= " $path_add" if ($path_add);
62
63 # Output the document (to swish)
64 print <<EOF;
65 Path-Name: $path
66 Content-Length: $size
67 Last-Mtime: $mtime
68 Document-Type: HTML
69
70 EOF
71 print $contents;
72
73 }

Properties

Name Value
cvs2svn:cvs-rev 1.5
svn:executable *

  ViewVC Help
Powered by ViewVC 1.1.26