viewvc.cgi/perl/HouseSpider.pl

#!/usr/bin/perl -w
#

use strict;

my $home = "/home/dpavlin/private/home_html";

my @dirs = ( $home );

while (@dirs) {
        my $path = shift @dirs;
        opendir(DIR, $path) || die "can't opendir $path: $!";
        foreach (readdir(DIR)) {
                next if (/^\./);        # skip .dot files
                if (-d "$path/$_") {
                        push @dirs,"$path/$_";
                        next;
                }
                if (-f "$path/$_" && /\.html?$/i) {
                        my $file="$path/$_";

                        open(HTML,$file) || warn "can't open $file: $!";
                        my $html = "";
                        while(<HTML>) {
                                chomp;
                                $html .= "$_ ";
                        }
                        close(HTML);

                        my $title = $file;
                        $title = $1 if ($html =~ m/<title[^>]*>([^<]+)<\/title>/i);

                        $html =~ s/<\/?[^>]+>//g;
                        $html =~ s/\s+/ /g;

                        # remove noindex
                        $html =~ s,<noindex>.+?</noindex>,,isg;
                        $html =~ s,<!--\s*noindex\s*-->.+?<!--\s*/noindex\s*-->,,isg;
                        # remove all script from indexing content
                        $html =~ s,<script>.+?</script>,,isg;

                        my $url = $file;
                        $url =~ s/^$home//;
                        $url =~ s/^\///;

                        print "$html\n$title\n$url\n";
                }
        }
        closedir DIR;
}

1	dpavlin	1.1	#!/usr/bin/perl -w
2			#
3
4			use strict;
5
6			my $home = "/home/dpavlin/private/home_html";
7
8			my @dirs = ( $home );
9
10			while (@dirs) {
11			my $path = shift @dirs;
12			opendir(DIR, $path) \|\| die "can't opendir $path: $!";
13			foreach (readdir(DIR)) {
14			next if (/^\./); # skip .dot files
15			if (-d "$path/$_") {
16			push @dirs,"$path/$_";
17			next;
18			}
19			if (-f "$path/$_" && /\.html?$/i) {
20			my $file="$path/$_";
21
22			open(HTML,$file) \|\| warn "can't open $file: $!";
23			my $html = "";
24			while(<HTML>) {
25			chomp;
26			$html .= "$_ ";
27			}
28			close(HTML);
29
30			my $title = $file;
31			$title = $1 if ($html =~ m/<title[^>]*>([^<]+)<\/title>/i);
32
33			$html =~ s/<\/?[^>]+>//g;
34			$html =~ s/\s+/ /g;
35
36			# remove noindex
37			$html =~ s,<noindex>.+?</noindex>,,isg;
38			$html =~ s,<!--\snoindex\s-->.+?<!--\s/noindex\s-->,,isg;
39			# remove all script from indexing content
40			$html =~ s,<script>.+?</script>,,isg;
41
42			my $url = $file;
43			$url =~ s/^$home//;
44			$url =~ s/^\///;
45
46			print "$html\n$title\n$url\n";
47			}
48			}
49			closedir DIR;
50			}
51