--- trunk2/lib/WebPAC.pm	2004/09/09 18:08:38	418
+++ trunk2/lib/WebPAC.pm	2004/09/10 22:24:42	421
@@ -29,20 +29,38 @@
 
 =head2 new
 
-This will create new instance of WebPAC using configuration specified by C<config_file>.
+Create new instance of WebPAC using configuration specified by C<config_file>.
 
  my $webpac = new WebPAC(
  	config_file => 'name.conf',
 	[code_page => 'ISO-8859-2',]
+	[low_mem => 1,]
  );
 
 Default C<code_page> is C<ISO-8859-2>.
 
-It will also read configuration files
+This method will also read configuration files
 C<global.conf> (used by indexer and Web font-end)
 and configuration file specified by C<config_file>
 which describes databases to be indexed.
 
+C<low_mem> options is double-edged sword. If enabled, WebPAC
+will run on memory constraint machines (which doesn't have enough
+physical RAM to create memory structure for whole ISIS database).
+
+If your machine has 512Mb or more and database is around 10000 records,
+memory shouldn't be an issue. If you don't have enough physical RAM, you
+might consider using virtual memory (if your operating system is handling it
+well, like on FreeBSD or Linux) instead of dropping to L<DBD::Deep> to handle
+parsed structure of ISIS database.
+
+However, when WebPAC is running on desktop machines (or laptops :-), it's
+highly undesireable for system to start swapping. Using C<low_mem> option can
+reduce WecPAC memory usage to 16Mb for same database with lookup fields and
+sorted indexes which stay in RAM. Performance will suffer, but memory usage
+will really be minimal. It might be also more confortable to run WebPAC reniced
+on those machines.
+
 =cut
 
 # mapping between data type and tag which specify
@@ -106,6 +124,32 @@
 		EVAL_PERL => 1,
 	);
 
+	# running with low_mem flag? well, use DBM::Deep then.
+	if ($self->{'low_mem'}) {
+		$log->info("running with low_mem which impacts performance (<64 Mb memory usage)");
+
+		my $db_file = "data.db";
+
+		if (-e $db_file) {
+			unlink $db_file or $log->logdie("can't remove '$db_file' from last run");
+			$log->debug("removed '$db_file' from last run");
+		}
+
+		use DBM::Deep;
+
+		my $db = new DBM::Deep $db_file;
+
+		$log->logdie("DBM::Deep error: $!") unless ($db);
+
+		if ($db->error()) {
+			$log->logdie("can't open '$db_file' under low_mem: ",$db->error());
+		} else {
+			$log->debug("using file $db_file for DBM::Deep");
+		}
+
+		$self->{'db'} = $db;
+	}
+
 	return $self;
 }
 
@@ -149,6 +193,10 @@
 	$log->logcroak("need filename") if (! $arg->{'filename'});
 	my $code_page = $arg->{'code_page'} || '852';
 
+	# store data in object
+	$self->{'isis_filename'} = $arg->{'filename'};
+	$self->{'isis_code_page'} = $code_page;
+
 	use OpenIsis;
 
 	#$self->{'isis_code_page'} = $code_page;
@@ -157,6 +205,7 @@
 	my $cp = Text::Iconv->new($code_page,$self->{'code_page'});
 
 	$log->info("reading ISIS database '",$arg->{'filename'},"'");
+	$log->debug("isis code page: $code_page");
 
 	my $isis_db = OpenIsis::open($arg->{'filename'});
 
@@ -172,6 +221,8 @@
 
 		$log->debug("mfn: $mfn\n");
 
+		my $rec;
+
 		# read record
 		my $row = OpenIsis::read( $isis_db, $mfn );
 		foreach my $k (keys %{$row}) {
@@ -189,16 +240,24 @@
 						$val = $l;
 					}
 
-					push @{$self->{'data'}->{$mfn}->{$k}}, $val;
+					push @{$rec->{$k}}, $val;
 				}
 			} else {
-				push @{$self->{'data'}->{$mfn}->{'000'}}, $mfn;
+				push @{$rec->{'000'}}, $mfn;
 			}
 
 		}
 
+		$log->confess("record $mfn empty?") unless ($rec);
+
+		# store
+		if ($self->{'low_mem'}) {
+			$self->{'db'}->put($mfn, $rec);
+		} else {
+			$self->{'data'}->{$mfn} = $rec;
+		}
+
 		# create lookup
-		my $rec = $self->{'data'}->{$mfn} || $log->confess("record $mfn empty?");
 		$self->create_lookup($rec, @{$arg->{'lookup'}});
 
 		$self->progress_bar($mfn,$maxmfn);
@@ -238,7 +297,11 @@
 
 	$self->progress_bar($mfn,$self->{'max_mfn'});
 
-	return $self->{'data'}->{$mfn};
+	if ($self->{'low_mem'}) {
+		return $self->{'db'}->get($mfn);
+	} else {
+		return $self->{'data'}->{$mfn};
+	}
 }
 
 =head2 progress_bar
@@ -803,11 +866,11 @@
 
 	my $log = $self->_get_logger();
 
-	$log->logconfess("need file name") if (! $args->{'file'});
+	my $file = $args->{'file'} || $log->logconfess("need file name");
 
-	$log->debug("creating file ",$args->{'file'});
+	$log->debug("creating file ",$file);
 
-	open(my $fh, ">", $args->{'file'}) || $log->logdie("can't open output file '$self->{'file'}': $!");
+	open(my $fh, ">", $file) || $log->logdie("can't open output file '$file': $!");
 	print $fh $self->output(
 		template => $args->{'template'},
 		data => $args->{'data'},