viewvc.cgi/perl/HouseSpider.pl

#!/usr/bin/perl -w
#

use strict;

my $home = "/home/dpavlin/private/home_html";

my @dirs = ( $home );

while (@dirs) {
        my $path = shift @dirs;
        opendir(DIR, $path) || die "can't opendir $path: $!";
        foreach (readdir(DIR)) {
                next if (/^\./);        # skip .dot files
                if (-d "$path/$_") {
                        push @dirs,"$path/$_";
                        next;
                }
                if (-f "$path/$_" && /\.html?$/i) {
                        my $file="$path/$_";

                        open(HTML,$file) || warn "can't open $file: $!";
                        my $html = "";
                        while(<HTML>) {
                                chomp;
                                $html .= "$_ ";
                        }
                        close(HTML);

                        my $title = $file;
                        $title = $1 if ($html =~ m/<title[^>]*>([^<]+)<\/title>/i);

                        $html =~ s/<\/?[^>]+>//g;
                        $html =~ s/\s+/ /g;

                        # remove noindex
                        $html =~ s,<noindex>.+?</noindex>,,isg;
                        $html =~ s,<!--\s*noindex\s*-->.+?<!--\s*/noindex\s*-->,,isg;
                        # remove all script from indexing content
                        $html =~ s,<script>.+?</script>,,isg;

                        my $url = $file;
                        $url =~ s/^$home//;
                        $url =~ s/^\///;

                        print "$html\n$title\n$url\n";
                }
        }
        closedir DIR;
}

1	#!/usr/bin/perl -w
2	#
3
4	use strict;
5
6	my $home = "/home/dpavlin/private/home_html";
7
8	my @dirs = ( $home );
9
10	while (@dirs) {
11	my $path = shift @dirs;
12	opendir(DIR, $path) \|\| die "can't opendir $path: $!";
13	foreach (readdir(DIR)) {
14	next if (/^\./); # skip .dot files
15	if (-d "$path/$_") {
16	push @dirs,"$path/$_";
17	next;
18	}
19	if (-f "$path/$_" && /\.html?$/i) {
20	my $file="$path/$_";
21
22	open(HTML,$file) \|\| warn "can't open $file: $!";
23	my $html = "";
24	while(<HTML>) {
25	chomp;
26	$html .= "$_ ";
27	}
28	close(HTML);
29
30	my $title = $file;
31	$title = $1 if ($html =~ m/<title[^>]*>([^<]+)<\/title>/i);
32
33	$html =~ s/<\/?[^>]+>//g;
34	$html =~ s/\s+/ /g;
35
36	# remove noindex
37	$html =~ s,<noindex>.+?</noindex>,,isg;
38	$html =~ s,<!--\snoindex\s-->.+?<!--\s/noindex\s-->,,isg;
39	# remove all script from indexing content
40	$html =~ s,<script>.+?</script>,,isg;
41
42	my $url = $file;
43	$url =~ s/^$home//;
44	$url =~ s/^\///;
45
46	print "$html\n$title\n$url\n";
47	}
48	}
49	closedir DIR;
50	}
51