/[swish]/trunk/spider/swishspider
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk/spider/swishspider

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 30 by dpavlin, Mon Mar 24 09:57:44 2003 UTC revision 42 by dpavlin, Tue Jul 29 10:40:58 2003 UTC
# Line 18  my $url = shift; Line 18  my $url = shift;
18    
19  my $no_parent_url;  my $no_parent_url;
20  if ($url =~ m/\s/) {  if ($url =~ m/\s/) {
21          ($url,$no_parent_url) = split(/\s/,$url,2);          ($no_parent_url,$url) = split(/\s/,$url,2);
22            # old scheme had URL, no parent and new is reverse
23            ($url,$no_parent_url) = ($no_parent_url,$url) if ($no_parent_url =~ m/\Q$url\E/);
24  }  }
25    
26  my $request = new HTTP::Request( "GET", $url );  my $request = new HTTP::Request( "GET", $url );
# Line 39  if( $response->code() == RC_OK ) { Line 41  if( $response->code() == RC_OK ) {
41          if ($no_parent_url) {          if ($no_parent_url) {
42                  if ($link =~ m/$no_parent_url/) {                  if ($link =~ m/$no_parent_url/) {
43                          # if this URL is below parent URL o.k....                          # if this URL is below parent URL o.k....
44                          print RESP "$link $no_parent_url\n";                          print RESP "$no_parent_url $link\n";
45                  } else {                  } else {
46                          # if not, crawl just this page!                          # if not, crawl just this page!
47                          print RESP "$link $link\n";                          print RESP "$link $link\n";
# Line 59  if( $response->code() == RC_OK ) { Line 61  if( $response->code() == RC_OK ) {
61      my $contents = $response->content();      my $contents = $response->content();
62    
63      open( CONTENTS, ">$localpath.contents" ) || die( "Could not open contents file $localpath.contents\n" );      open( CONTENTS, ">$localpath.contents" ) || die( "Could not open contents file $localpath.contents\n" );
64      # if you don't want content to be indexed, include it in      # fixup just HTML files
65      # <noindex> foobar </noindex> tags or surround it with comments      if ($response->header("content-type") =~ "text/html") {
66      # <!-- noindex --> foobar <!-- /noindex -->          # if you don't want content to be indexed, include it in
67      $contents =~ s,<noindex>.+?</noindex>,,isg;          # <noindex> foobar </noindex> tags or surround it with comments
68      $contents =~ s,<!--\s*noindex\s*-->.+?<!--\s*/noindex\s*-->,,isg;          # <!-- noindex --> foobar <!-- /noindex -->
69      # this will remove all script from indexing content          # <!-- noindex --> foobar <!-- index --> (also supported by swish)
70      $contents =~ s,<script>.+?</script>,,isg;          $contents =~ s,<noindex>.+?</noindex>,,isg;
71      # remap Windows charset to ISO-8859-2          $contents =~ s,<!--\s*noindex\s*-->.+?<!--\s*/noindex\s*-->,,isg;
72      $contents =~ tr/Ў/Ю/;     # 1250 -> iso8859-2          $contents =~ s,<!--\s*noindex\s*-->.+?<!--\s*index\s*-->,,isg;
73      # this will fix badly formatted html in form:          # this will remove all script from indexing content
74      # <head><title>some text</title          $contents =~ s,<script>.+?</script>,,isg;
75      # ></head>          # remap Windows charset to ISO-8859-2
76      # which will confuse indexer (or libxml2?)          $contents =~ tr/Ў/Ю/; # 1250 -> iso8859-2
77      $contents =~ s/[\n\r]^(>)/$1\n/msg;          # this will fix badly formatted html in form:
78            # <head><title>some text</title
79            # ></head>
80            # which will confuse indexer (or libxml2?)
81            $contents =~ s/[\n\r]^(>)/$1\n/msg;
82            # remove comments between <html> and <head> texi2html inserts them
83            # there and swish can't find document title then (libxml or swish bug?)
84            while ($contents =~ s/(<html>.*)<!--.*?-->(.*<head>)/$1$2/msi) { };
85        }
86      print CONTENTS $contents;      print CONTENTS $contents;
87      close( CONTENTS );      close( CONTENTS );
88    
# Line 88  if( $response->code() == RC_OK ) { Line 98  if( $response->code() == RC_OK ) {
98    
99  sub linkcb {  sub linkcb {
100      my($tag, %links) = @_;      my($tag, %links) = @_;
101      if (($tag eq "a" || $tag eq "area") && ($links{"href"})) {      if (($tag eq "a" || $tag eq "area") && ($links{"href"}) || ($tag eq "frame" && $links{"src"})) {
102          my $link = $links{"href"};          my $link = $links{"href"} || $links{"src"};
103    
104          #          #
105          # Remove fragments          # Remove fragments
# Line 113  sub linkcb { Line 123  sub linkcb {
123    
124          if ($no_parent_url) {          if ($no_parent_url) {
125                          if ($link =~ m/$no_parent_url/) {                          if ($link =~ m/$no_parent_url/) {
126                                  print LINKS "$link $no_parent_url\n";                                  print LINKS "$no_parent_url $link\n";
127  #                               print STDERR "using $link\n";  #                               print STDERR "using $link\n";
128  #                       } else {  #                       } else {
129  #                               print STDERR "skipping $link\n";  #                               print STDERR "skipping $link\n";

Legend:
Removed from v.30  
changed lines
  Added in v.42

  ViewVC Help
Powered by ViewVC 1.1.26