/[webpac]/trunk2/lib/WebPAC.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /trunk2/lib/WebPAC.pm

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 421 by dpavlin, Fri Sep 10 22:24:42 2004 UTC revision 422 by dpavlin, Sat Sep 11 08:36:38 2004 UTC
# Line 9  use Config::IniFiles; Line 9  use Config::IniFiles;
9  use XML::Simple;  use XML::Simple;
10  use Template;  use Template;
11  use Log::Log4perl qw(get_logger :levels);  use Log::Log4perl qw(get_logger :levels);
12    use Time::HiRes qw(time);
13    
14  use Data::Dumper;  use Data::Dumper;
15    
# Line 39  Create new instance of WebPAC using conf Line 40  Create new instance of WebPAC using conf
40    
41  Default C<code_page> is C<ISO-8859-2>.  Default C<code_page> is C<ISO-8859-2>.
42    
43    Default is not to use C<low_mem> options (see L<MEMORY USAGE> below).
44    
45  This method will also read configuration files  This method will also read configuration files
46  C<global.conf> (used by indexer and Web font-end)  C<global.conf> (used by indexer and Web font-end)
47  and configuration file specified by C<config_file>  and configuration file specified by C<config_file>
48  which describes databases to be indexed.  which describes databases to be indexed.
49    
 C<low_mem> options is double-edged sword. If enabled, WebPAC  
 will run on memory constraint machines (which doesn't have enough  
 physical RAM to create memory structure for whole ISIS database).  
   
 If your machine has 512Mb or more and database is around 10000 records,  
 memory shouldn't be an issue. If you don't have enough physical RAM, you  
 might consider using virtual memory (if your operating system is handling it  
 well, like on FreeBSD or Linux) instead of dropping to L<DBD::Deep> to handle  
 parsed structure of ISIS database.  
   
 However, when WebPAC is running on desktop machines (or laptops :-), it's  
 highly undesireable for system to start swapping. Using C<low_mem> option can  
 reduce WecPAC memory usage to 16Mb for same database with lookup fields and  
 sorted indexes which stay in RAM. Performance will suffer, but memory usage  
 will really be minimal. It might be also more confortable to run WebPAC reniced  
 on those machines.  
   
50  =cut  =cut
51    
52  # mapping between data type and tag which specify  # mapping between data type and tag which specify
# Line 77  sub new { Line 63  sub new {
63          my $self = {@_};          my $self = {@_};
64          bless($self, $class);          bless($self, $class);
65    
66            $self->{'start_t'} = time();
67    
68          my $log_file = $self->{'log'} || "log.conf";          my $log_file = $self->{'log'} || "log.conf";
69          Log::Log4perl->init($log_file);          Log::Log4perl->init($log_file);
70    
# Line 126  sub new { Line 114  sub new {
114    
115          # running with low_mem flag? well, use DBM::Deep then.          # running with low_mem flag? well, use DBM::Deep then.
116          if ($self->{'low_mem'}) {          if ($self->{'low_mem'}) {
117                  $log->info("running with low_mem which impacts performance (<64 Mb memory usage)");                  $log->info("running with low_mem which impacts performance (<32 Mb memory usage)");
118    
119                  my $db_file = "data.db";                  my $db_file = "data.db";
120    
# Line 144  sub new { Line 132  sub new {
132                  if ($db->error()) {                  if ($db->error()) {
133                          $log->logdie("can't open '$db_file' under low_mem: ",$db->error());                          $log->logdie("can't open '$db_file' under low_mem: ",$db->error());
134                  } else {                  } else {
135                          $log->debug("using file $db_file for DBM::Deep");                          $log->debug("using file '$db_file' for DBM::Deep");
136                  }                  }
137    
138                  $self->{'db'} = $db;                  $self->{'db'} = $db;
# Line 329  sub progress_bar { Line 317  sub progress_bar {
317    
318          $self->{'last_pcnt'} ||= 1;          $self->{'last_pcnt'} ||= 1;
319    
         $self->{'last_pcnt'} = $curr if ($curr < $self->{'last_pcnt'});  
   
320          my $p = int($curr * 100 / $max);          my $p = int($curr * 100 / $max);
321    
322            # reset on re-run
323            if ($p < $self->{'last_pcnt'}) {
324                    $self->{'last_pcnt'} = $p;
325                    $self->{'last_t'} = time();
326                    $self->{'last_curr'} = 1;
327            }
328    
329          if ($p != $self->{'last_pcnt'}) {          if ($p != $self->{'last_pcnt'}) {
330                  printf STDERR ("%5d / %5d [%-51s] %-2d %% \r",$curr,$max,"=" x ($p/2).">", $p );  
331                    my $last_curr = $self->{'last_curr'} || $curr;
332                    my $t = time();
333                    my $rate = ($curr - $last_curr) / (($t - $self->{'last_t'} || 1));
334                    my $eta = ($max-$curr) / ($rate || 1);
335                    printf STDERR ("%5d [%-38s] %-5d %0.1f/s %s\r",$curr,"=" x ($p/3)."$p%>", $max, $rate, $self->fmt_time($eta));
336                  $self->{'last_pcnt'} = $p;                  $self->{'last_pcnt'} = $p;
337                    $self->{'last_t'} = time();
338                    $self->{'last_curr'} = $curr;
339          }          }
340          print STDERR "\n" if ($p == 100);          print STDERR "\n" if ($p == 100);
341  }  }
342    
343    =head2 fmt_time
344    
345    Format time (in seconds) for display.
346    
347     print $webpac->fmt_time(time());
348    
349    This method is called by L<progress_bar> to display remaining time.
350    
351    =cut
352    
353    sub fmt_time {
354            my $self = shift;
355    
356            my $t = shift || 0;
357            my $out = "";
358    
359            my ($ss,$mm,$hh) = gmtime($t);
360            $out .= "${hh}h" if ($hh);
361            $out .= sprintf("%02d:%02d", $mm,$ss);
362            $out .= "  " if ($hh == 0);
363            return $out;
364    }
365    
366  =head2 open_import_xml  =head2 open_import_xml
367    
368  Read file from C<import_xml/> directory and parse it.  Read file from C<import_xml/> directory and parse it.
# Line 1021  B<This is different from normal Log4perl Line 1045  B<This is different from normal Log4perl
1045  also use method names, and not only classes (which are just few)  also use method names, and not only classes (which are just few)
1046  to filter logging.  to filter logging.
1047    
1048    
1049    =head1 MEMORY USAGE
1050    
1051    C<low_mem> options is double-edged sword. If enabled, WebPAC
1052    will run on memory constraint machines (which doesn't have enough
1053    physical RAM to create memory structure for whole source database).
1054    
1055    If your machine has 512Mb or more of RAM and database is around 10000 records,
1056    memory shouldn't be an issue. If you don't have enough physical RAM, you
1057    might consider using virtual memory (if your operating system is handling it
1058    well, like on FreeBSD or Linux) instead of dropping to L<DBD::Deep> to handle
1059    parsed structure of ISIS database (this is what C<low_mem> option does).
1060    
1061    Hitting swap at end of reading source database is probably o.k. However,
1062    hitting swap before 90% will dramatically decrease performance and you will
1063    be better off with C<low_mem> and using rest of availble memory for
1064    operating system disk cache (Linux is particuallary good about this).
1065    However, every access to database record will require disk access, so
1066    generation phase will be slower 10-100 times.
1067    
1068    Parsed structures are essential - you just have option to trade RAM memory
1069    (which is fast) for disk space (which is slow). Be sure to have planty of
1070    disk space if you are using C<low_mem> and thus L<DBD::Deep>.
1071    
1072    However, when WebPAC is running on desktop machines (or laptops :-), it's
1073    highly undesireable for system to start swapping. Using C<low_mem> option can
1074    reduce WecPAC memory usage to around 64Mb for same database with lookup
1075    fields and sorted indexes which stay in RAM. Performance will suffer, but
1076    memory usage will really be minimal. It might be also more confortable to
1077    run WebPAC reniced on those machines.
1078    
1079  =cut  =cut
1080    
1081  1;  1;

Legend:
Removed from v.421  
changed lines
  Added in v.422

  ViewVC Help
Powered by ViewVC 1.1.26