/[BackupPC]/trunk/lib/BackupPC/SearchLib.pm
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /trunk/lib/BackupPC/SearchLib.pm

Parent Directory Parent Directory | Revision Log Revision Log


Revision 122 - (show annotations)
Fri Sep 16 14:11:00 2005 UTC (18 years, 8 months ago) by dpavlin
File size: 15422 byte(s)
1000x speedup (60 s -> 60 ms)

1 #!/usr/bin/perl
2 package BackupPC::SearchLib;
3
4 use strict;
5 use BackupPC::CGI::Lib qw(:all);
6 use BackupPC::Attrib qw(:all);
7 use DBI;
8 use DateTime;
9 use vars qw(%In $MyURL);
10 use Time::HiRes qw/time/;
11
12 my $on_page = 100;
13 my $pager_pages = 10;
14
15 my $dsn = $Conf{SearchDSN};
16 my $db_user = $Conf{SearchUser} || '';
17
18 my $index_path = $Conf{HyperEstraierIndex};
19 if ($index_path) {
20 $index_path = $TopDir . '/' . $index_path;
21 $index_path =~ s#//#/#g;
22 }
23
24 my $dbh;
25
26 sub get_dbh {
27 $dbh ||= DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1 } );
28 return $dbh;
29 }
30
31 sub getUnits() {
32 my @ret;
33
34 my $dbh = get_dbh();
35 my $sth = $dbh->prepare(qq{
36 SELECT
37 shares.id as id,
38 hosts.name || ':' || shares.name as share
39 FROM shares
40 JOIN hosts on hostid = hosts.id
41 ORDER BY share
42 } );
43 $sth->execute();
44 push @ret, { 'id' => '', 'share' => '-'}; # dummy any
45
46 while ( my $row = $sth->fetchrow_hashref() ) {
47 push @ret, $row;
48 }
49 return @ret;
50 }
51
52 sub epoch_to_iso {
53 my $t = shift || return;
54 my $iso = BackupPC::Lib::timeStamp(undef, $t);
55 $iso =~ s/\s/ /g;
56 return $iso;
57 }
58
59 sub dates_from_form($) {
60 my $param = shift || return;
61
62 sub mk_epoch_date($$) {
63 my ($name,$suffix) = @_;
64
65 my $yyyy = $param->{ $name . '_year_' . $suffix} || return undef;
66 my $mm .= $param->{ $name . '_month_' . $suffix} ||
67 ( $suffix eq 'from' ? 1 : 12);
68 my $dd .= $param->{ $name . '_day_' . $suffix} ||
69 ( $suffix eq 'from' ? 1 : 31);
70
71 $yyyy =~ s/\D//g;
72 $mm =~ s/\D//g;
73 $dd =~ s/\D//g;
74
75 my $dt = new DateTime(
76 year => $yyyy,
77 month => $mm,
78 day => $dd
79 );
80 print STDERR "mk_epoch_date($name,$suffix) [$yyyy-$mm-$dd] = " . $dt->ymd . " " . $dt->hms . "\n";
81 return $dt->epoch || 'NULL';
82 }
83
84 my @ret = (
85 mk_epoch_date('search_backup', 'from'),
86 mk_epoch_date('search_backup', 'to'),
87 mk_epoch_date('search', 'from'),
88 mk_epoch_date('search', 'to'),
89 );
90
91 return @ret;
92
93 }
94
95
96 sub getWhere($) {
97 my $param = shift || return;
98
99 my ($backup_from, $backup_to, $files_from, $files_to) = dates_from_form($param);
100
101 my @conditions;
102 push @conditions, qq{ backups.date >= $backup_from } if ($backup_from);
103 push @conditions, qq{ backups.date <= $backup_to } if ($backup_to);
104 push @conditions, qq{ files.date >= $files_from } if ($files_from);
105 push @conditions, qq{ files.date <= $files_to } if ($files_to);
106
107 print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" | ",@conditions);
108
109 push( @conditions, ' files.shareid = ' . $param->{'search_share'} ) if ($param->{'search_share'});
110 push (@conditions, " upper(files.path) LIKE upper('%".$param->{'search_filename'}."%')") if ($param->{'search_filename'});
111
112 return join(" and ", @conditions);
113 }
114
115
116 sub getFiles($) {
117 my ($param) = @_;
118
119 my $offset = $param->{'offset'} || 0;
120 $offset *= $on_page;
121
122 my $dbh = get_dbh();
123
124 my $sql_cols = qq{
125 files.id AS fid,
126 hosts.name AS hname,
127 shares.name AS sname,
128 files.backupnum AS backupnum,
129 files.path AS filepath,
130 files.date AS date,
131 files.type AS type,
132 files.size AS size
133 };
134
135 my $sql_from = qq{
136 FROM files
137 INNER JOIN shares ON files.shareID=shares.ID
138 INNER JOIN hosts ON hosts.ID = shares.hostID
139 INNER JOIN backups ON backups.num = files.backupnum and backups.hostID = hosts.ID AND backups.shareID = files.shareID
140 };
141
142 my $sql_where;
143 my $where = getWhere($param);
144 $sql_where = " WHERE ". $where if ($where);
145
146 my $sql_order = qq{
147 ORDER BY files.date
148 LIMIT $on_page
149 OFFSET ?
150 };
151
152 my $sql_count = qq{ select count(files.id) $sql_from $sql_where };
153 my $sql_results = qq{ select $sql_cols $sql_from $sql_where $sql_order };
154
155 my $sth = $dbh->prepare($sql_count);
156 $sth->execute();
157 my ($results) = $sth->fetchrow_array();
158
159 $sth = $dbh->prepare($sql_results);
160 $sth->execute( $offset );
161
162 if ($sth->rows != $results) {
163 my $bug = "$0 BUG: [[ $sql_count ]] = $results while [[ $sql_results ]] = " . $sth->rows;
164 $bug =~ s/\s+/ /gs;
165 print STDERR "$bug\n";
166 }
167
168 my @ret;
169
170 while (my $row = $sth->fetchrow_hashref()) {
171 push @ret, $row;
172 }
173
174 $sth->finish();
175 return ($results, \@ret);
176 }
177
178 sub getHyperEstraier_url($) {
179 my ($use_hest) = @_;
180
181 return unless $use_hest;
182
183 use HyperEstraier;
184 my ($index_path, $index_node_url);
185
186 if ($use_hest =~ m#^http://#) {
187 $index_node_url = $use_hest;
188 } else {
189 $index_path = $TopDir . '/' . $index_path;
190 $index_path =~ s#//#/#g;
191 }
192 return ($index_path, $index_node_url);
193 }
194
195 sub getFilesHyperEstraier($) {
196 my ($param) = @_;
197
198 my $offset = $param->{'offset'} || 0;
199 $offset *= $on_page;
200
201 die "no index_path?" unless ($index_path);
202
203 use HyperEstraier;
204
205 my ($index_path, $index_node_url) = getHyperEstraier_url($index_path);
206
207 # open the database
208 my $db;
209 if ($index_path) {
210 $db = HyperEstraier::Database->new();
211 $db->open($index_path, $HyperEstraier::ESTDBREADER);
212 } elsif ($index_node_url) {
213 $db ||= HyperEstraier::Node->new($index_node_url);
214 $db->set_auth('admin', 'admin');
215 } else {
216 die "BUG: unimplemented";
217 }
218
219 # create a search condition object
220 my $cond = HyperEstraier::Condition->new();
221
222 my $q = $param->{'search_filename'};
223 my $shareid = $param->{'search_share'};
224
225 if (length($q) > 0) {
226 # exact match
227 $cond->add_attr("filepath ISTRINC $q");
228
229 $q =~ s/(.)/$1 /g;
230 # set the search phrase to the search condition object
231 $cond->set_phrase($q);
232 }
233
234 my ($backup_from, $backup_to, $files_from, $files_to) = dates_from_form($param);
235
236 $cond->add_attr("backup_date NUMGE $backup_from") if ($backup_from);
237 $cond->add_attr("backup_date NUMLE $backup_to") if ($backup_to);
238
239 $cond->add_attr("date NUMGE $files_from") if ($files_from);
240 $cond->add_attr("date NUMLE $files_to") if ($files_to);
241
242 $cond->add_attr("shareid NUMEQ $shareid") if ($shareid);
243
244 # $cond->set_max( $offset + $on_page );
245 $cond->set_options( $HyperEstraier::Condition::SURE );
246 $cond->set_order( 'date NUMA' );
247
248 # get the result of search
249 my @res;
250 my ($result, $hits);
251
252 if ($index_path) {
253 $result = $db->search($cond, 0);
254 $hits = $result->size;
255 } elsif ($index_node_url) {
256 $result = $db->search($cond, 0);
257 $hits = $result->doc_num;
258 } else {
259 die "BUG: unimplemented";
260 }
261
262 # for each document in result
263 for my $i ($offset .. ($offset + $on_page - 1)) {
264 last if ($i >= $hits);
265
266 my $doc;
267 if ($index_path) {
268 my $id = $result->get($i);
269 $doc = $db->get_doc($id, 0);
270 } elsif ($index_node_url) {
271 $doc = $result->get_doc($i);
272 } else {
273 die "BUG: unimplemented";
274 }
275
276 my $row;
277 foreach my $c (qw/fid hname sname backupnum fiilename filepath date type size/) {
278 $row->{$c} = $doc->attr($c);
279 }
280 push @res, $row;
281 }
282
283 return ($hits, \@res);
284 }
285
286 sub getGzipName($$$)
287 {
288 my ($host, $share, $backupnum) = @_;
289 my $ret = $Conf{GzipSchema};
290
291 $share =~ s/\//_/g;
292 $ret =~ s/\\h/$host/ge;
293 $ret =~ s/\\s/$share/ge;
294 $ret =~ s/\\n/$backupnum/ge;
295
296 return $ret;
297
298 }
299
300 sub getBackupsNotBurned() {
301
302 my $dbh = get_dbh();
303
304 my $sql = q{
305 SELECT
306 backups.hostID AS hostID,
307 hosts.name AS host,
308 shares.name AS share,
309 backups.id AS backupnum,
310 backups.type AS type,
311 backups.date AS date,
312 backups.size AS size
313 FROM backups
314 INNER JOIN shares ON backups.shareID=shares.ID
315 INNER JOIN hosts ON backups.hostID = hosts.ID
316 LEFT OUTER JOIN archive_backup ON archive_backup.backup_id = backups.id AND archive_backup.backup_id IS NULL
317 WHERE backups.size > 0
318 GROUP BY
319 backups.hostID,
320 hosts.name,
321 shares.name,
322 backups.num,
323 backups.shareid,
324 backups.id,
325 backups.type,
326 backups.date,
327 backups.size
328 ORDER BY backups.date
329 };
330 my $sth = $dbh->prepare( $sql );
331 my @ret;
332 $sth->execute();
333
334 while ( my $row = $sth->fetchrow_hashref() ) {
335 $row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) );
336 $row->{'size'} = sprintf("%0.2f", $row->{'size'} / 1024 / 1024);
337 my (undef,undef,undef,undef,undef,undef,undef,$fs_size,undef,undef,undef,undef,undef) =
338 stat( $Conf{InstallDir}.'/'.$Conf{GzipTempDir}.'/'.
339 getGzipName($row->{'host'}, $row->{share}, $row->{'backupnum'}));
340 $row->{'fs_size'} = $fs_size;
341 push @ret, $row;
342 }
343
344 return @ret;
345 }
346
347 sub displayBackupsGrid()
348 {
349 my $retHTML = "";
350
351 $retHTML .= <<EOF3;
352 <script language="javascript" type="text/javascript">
353 <!--
354
355 function checkAll(location)
356 {
357 for (var i=0;i<document.forma.elements.length;i++)
358 {
359 var e = document.forma.elements[i];
360 if ((e.checked || !e.checked) && e.name != \'all\') {
361 if (eval("document.forma."+location+".checked")) {
362 e.checked = true;
363 } else {
364 e.checked = false;
365 }
366 }
367 }
368 }
369
370 function sumiraj()
371 {
372 var suma = 0;
373 for (var i = 0; i < document.forma.elements.length; i++)
374 {
375 var e = document.forma.elements[i];
376 if ((e.checked || !e.checked) && e.name != \'all\')
377 {
378 if (e.checked)
379 {
380 var ret = e.name.match("fcb(.*)");
381 suma += parseInt(eval("document.forma.fss"+ret[1]+".value"));
382
383 }
384 }
385 }
386 document.forma.totalsize.value = suma;
387 return suma;
388 }
389 //-->
390 </script>
391 EOF3
392 $retHTML .= q{
393 <form name="forma" method="GET" action=};
394 $retHTML .= "\"".$MyURL."\"";
395 $retHTML .= q{?action=burn>
396 <input type="hidden" value="burn" name="action">
397 <input type="hidden" value="results" name="search_results">
398 <table style="fview" border="0" cellspacing="0" cellpadding="2">
399 <tr class="tableheader">
400 <td class="tableheader">
401 <input type="checkbox" name="allFiles" onClick="checkAll('allFiles');">
402 </td>
403 <td align="center">Share</td>
404 <td align="center">Backup no</td>
405 <td align="center">Type</td>
406 <td align="center">date</td>
407 <td align="center">age/days</td>
408 <td align="center">size/MB</td>
409 <td align="center">gzip size</td>
410 </tr>
411
412 <tr><td colspan=7 style="tableheader">
413 <input type="submit" value="Burn selected backups on medium" name="submitBurner">
414 </td></tr>
415 };
416
417 my @color = (' bgcolor="#e0e0e0"', '');
418
419 my $i = 0;
420 my $host = '';
421
422 foreach my $backup ( getBackupsNotBurned() ) {
423
424 if ($host ne $backup->{'host'}) {
425 $i++;
426 $host = $backup->{'host'};
427 }
428 my $ftype = "";
429
430 $retHTML .= "<tr" . $color[$i %2 ] . ">";
431 $retHTML .= '<td class="fview"><input type="checkbox" name="fcb' .
432 $backup->{'hostid'}.'_'.$backup->{'backupnum'} .
433 '" value="' . $backup->{'hostid'}.'_'.$backup->{'backupnum'} .
434 '" onClick="sumiraj();"></td>';
435
436 $retHTML .=
437 '<td align="right">' . $backup->{'host'} . ':' . $backup->{'share'} . '</td>' .
438 '<td align="center">' . $backup->{'backupnum'} . '</td>' .
439 '<td align="center">' . $backup->{'type'} . '</td>' .
440 '<td align="center">' . epoch_to_iso( $backup->{'date'} ) . '</td>' .
441 '<td align="center">' . $backup->{'age'} . '</td>' .
442 '<td align="right">' . $backup->{'size'} . '</td>' .
443 '<td align="right">' . $backup->{'fs_size'} .
444 '<input type="hidden" name="fss'.$backup->{'hostid'}.'_'.$backup->{'backupnum'} . '"'.
445 'value="'. $backup->{'fs_size'} .'"'.'</td>' .
446 "</tr>\n";
447
448
449 }
450
451 $retHTML .= "</table>";
452 $retHTML .= "total gzip size:<input type=\"text\" name=\"totalsize\"><br>";
453 $retHTML .= "Note:<input type=\"text\" name=\"note\">";
454 $retHTML .= "</form>";
455
456 return $retHTML;
457 }
458
459 sub displayGrid($) {
460 my ($param) = @_;
461
462 my $offset = $param->{'offset'};
463 my $hilite = $param->{'search_filename'};
464
465 my $retHTML = "";
466
467 my $start_t = time();
468
469 my ($results, $files);
470 if ($param->{'use_hest'} && length($hilite) > 0) {
471 ($results, $files) = getFilesHyperEstraier($param);
472 } else {
473 ($results, $files) = getFiles($param);
474 }
475
476 my $dur_t = time() - $start_t;
477 my $dur = sprintf("%0.4fs", $dur_t);
478
479 my ($from, $to) = (($offset * $on_page) + 1, ($offset * $on_page) + $on_page);
480
481 if ($results <= 0) {
482 $retHTML .= qq{
483 <p style="color: red;">No results found...</p>
484 };
485 return $retHTML;
486 } else {
487 # DEBUG
488 #use Data::Dumper;
489 #$retHTML .= '<pre>' . Dumper($files) . '</pre>';
490 }
491
492
493 $retHTML .= qq{
494 <div>
495 Found <b>$results files</b> showing <b>$from - $to</b> (took $dur)
496 </div>
497 <table style="fview" width="100%" border="0" cellpadding="2" cellspacing="0">
498 <tr class="fviewheader">
499 <td></td>
500 <td align="center">Share</td>
501 <td align="center">Type and Name</td>
502 <td align="center">#</td>
503 <td align="center">Size</td>
504 <td align="center">Date</td>
505 <td align="center">Media</td>
506 </tr>
507 };
508
509 my $file;
510
511 sub hilite_html($$) {
512 my ($html, $search) = @_;
513 $html =~ s#($search)#<b>$1</b>#gis;
514 return $html;
515 }
516
517 sub restore_link($$$$$$) {
518 my $type = shift;
519 my $action = 'RestoreFile';
520 $action = 'browse' if (lc($type) eq 'dir');
521 return sprintf(qq{<a href="?action=%s&host=%s&num=%d&share=%s&dir=%s">%s</a>}, $action, @_);
522 }
523
524 my $i = $offset * $on_page;
525
526 foreach $file (@{ $files }) {
527 $i++;
528
529 my $typeStr = BackupPC::Attrib::fileType2Text(undef, $file->{'type'});
530 $retHTML .= qq{<tr class="fviewborder">};
531
532 $retHTML .= qq{<td class="fviewborder">$i</td>};
533
534 $retHTML .=
535 qq{<td class="fviewborder" align="right">} . $file->{'hname'} . ':' . $file->{'sname'} . qq{</td>} .
536 qq{<td class="fviewborder"><img src="$Conf{CgiImageDirURL}/icon-$typeStr.gif" alt="$typeStr" align="middle">&nbsp;} . hilite_html( $file->{'filepath'}, $hilite ) . qq{</td>} .
537 qq{<td class="fviewborder" align="center">} . restore_link( $typeStr, ${EscURI( $file->{'hname'} )}, $file->{'backupnum'}, ${EscURI( $file->{'sname'})}, ${EscURI( $file->{'filepath'} )}, $file->{'backupnum'} ) . qq{</td>} .
538 qq{<td class="fviewborder" align="right">} . $file->{'size'} . qq{</td>} .
539 qq{<td class="fviewborder">} . epoch_to_iso( $file->{'date'} ) . qq{</td>} .
540 qq{<td class="fviewborder">} . '?' . qq{</td>};
541
542 $retHTML .= "</tr>";
543 }
544 $retHTML .= "</table>";
545
546 # all variables which has to be transfered
547 foreach my $n (qw/search_day_from search_month_from search_year_from search_day_to search_month_to search_year_to search_backup_day_from search_backup_month_from search_backup_year_from search_backup_day_to search_backup_month_to search_backup_year_to search_filename offset/) {
548 $retHTML .= qq{<INPUT TYPE="hidden" NAME="$n" VALUE="$In{$n}">\n};
549 }
550
551 my $del = '';
552 my $max_page = int( $results / $on_page );
553 my $page = 0;
554
555 sub page_link($$$) {
556 my ($param,$page,$display) = @_;
557
558 $param->{'offset'} = $page;
559
560 my $html = '<a href = "' . $MyURL;
561 my $del = '?';
562 foreach my $k (keys %{ $param }) {
563 if ($param->{$k}) {
564 $html .= $del . $k . '=' . ${EscURI( $param->{$k} )};
565 $del = '&';
566 }
567 }
568 $html .= '">' . $display . '</a>';
569 }
570
571 $retHTML .= '<div style="text-align: center;">';
572
573 if ($offset > 0) {
574 $retHTML .= page_link($param, $offset - 1, '&lt;&lt;') . ' ';
575 }
576
577 while ($page <= $max_page) {
578 if ($page == $offset) {
579 $retHTML .= $del . '<b>' . ($page + 1) . '</b>';
580 } else {
581 $retHTML .= $del . page_link($param, $page, $page + 1);
582 }
583
584 if ($page < $offset - $pager_pages && $page != 0) {
585 $retHTML .= " ... ";
586 $page = $offset - $pager_pages;
587 $del = '';
588 } elsif ($page > $offset + $pager_pages && $page != $max_page) {
589 $retHTML .= " ... ";
590 $page = $max_page;
591 $del = '';
592 } else {
593 $del = ' | ';
594 $page++;
595 }
596 }
597
598 if ($offset < $max_page) {
599 $retHTML .= ' ' . page_link($param, $offset + 1, '&gt;&gt;');
600 }
601
602 $retHTML .= "</div>";
603
604 return $retHTML;
605 }
606
607 1;

  ViewVC Help
Powered by ViewVC 1.1.26