--- trunk/lib/BackupPC/SearchLib.pm 2005/08/28 17:53:46 91 +++ trunk/lib/BackupPC/SearchLib.pm 2005/10/24 15:04:45 227 @@ -8,6 +8,8 @@ use DateTime; use vars qw(%In $MyURL); use Time::HiRes qw/time/; +use XML::Writer; +use IO::File; my $on_page = 100; my $pager_pages = 10; @@ -15,11 +17,7 @@ my $dsn = $Conf{SearchDSN}; my $db_user = $Conf{SearchUser} || ''; -my $index_path = $Conf{HyperEstraierIndex}; -if ($index_path) { - $index_path = $TopDir . '/' . $index_path; - $index_path =~ s#//#/#g; -} +my $hest_index_path = $Conf{HyperEstraierIndex}; my $dbh; @@ -72,10 +70,20 @@ $mm =~ s/\D//g; $dd =~ s/\D//g; + my $h = my $m = my $s = 0; + if ($suffix eq 'to') { + $h = 23; + $m = 59; + $s = 59; + } + my $dt = new DateTime( year => $yyyy, month => $mm, - day => $dd + day => $dd, + hour => $h, + minute => $m, + second => $s, ); print STDERR "mk_epoch_date($name,$suffix) [$yyyy-$mm-$dd] = " . $dt->ymd . " " . $dt->hms . "\n"; return $dt->epoch || 'NULL'; @@ -104,7 +112,7 @@ push @conditions, qq{ files.date >= $files_from } if ($files_from); push @conditions, qq{ files.date <= $files_to } if ($files_to); - print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" | ",@conditions); + print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" and ",@conditions); push( @conditions, ' files.shareid = ' . $param->{'search_share'} ) if ($param->{'search_share'}); push (@conditions, " upper(files.path) LIKE upper('%".$param->{'search_filename'}."%')") if ($param->{'search_filename'}); @@ -112,6 +120,67 @@ return join(" and ", @conditions); } +my $sort_def = { + search => { + default => 'date_a', + sql => { + share_d => 'shares.name DESC', + share_a => 'shares.name ASC', + path_d => 'files.path DESC', + path_a => 'files.path ASC', + num_d => 'files.backupnum DESC', + num_a => 'files.backupnum ASC', + size_d => 'files.size DESC', + size_a => 'files.size ASC', + date_d => 'files.date DESC', + date_a => 'files.date ASC', + }, + est => { + share_d => 'sname STRD', + share_a => 'sname STRA', + path_d => 'filepath STRD', + path_a => 'filepath STRA', + num_d => 'backupnum NUMD', + num_a => 'backupnum NUMA', + size_d => 'size NUMD', + size_a => 'size NUMA', + date_d => 'date NUMD', + date_a => 'date NUMA', + } + }, burn => { + default => 'date_a', + sql => { + share_d => 'host DESC, share DESC', + share_a => 'host ASC, share ASC', + num_d => 'backupnum DESC', + num_a => 'backupnum ASC', + date_d => 'date DESC', + date_a => 'date ASC', + age_d => 'age DESC', + age_a => 'age ASC', + size_d => 'size DESC', + size_a => 'size ASC', + incsize_d => 'inc_size DESC', + incsize_a => 'inc_size ASC', + } + } +}; + +sub getSort($$$) { + my ($part,$type, $sort_order) = @_; + + die "unknown part: $part" unless ($sort_def->{$part}); + die "unknown type: $type" unless ($sort_def->{$part}->{$type}); + + $sort_order ||= $sort_def->{$part}->{'default'}; + + if (my $ret = $sort_def->{$part}->{$type}->{$sort_order}) { + return $ret; + } else { + # fallback to default sort order + return $sort_def->{$part}->{$type}->{ $sort_def->{$part}->{'default'} }; + } +} sub getFiles($) { my ($param) = @_; @@ -143,8 +212,10 @@ my $where = getWhere($param); $sql_where = " WHERE ". $where if ($where); + my $order = getSort('search', 'sql', $param->{'sort'}); + my $sql_order = qq{ - ORDER BY files.date + ORDER BY $order LIMIT $on_page OFFSET ? }; @@ -175,19 +246,46 @@ return ($results, \@ret); } +sub getHyperEstraier_url($) { + my ($use_hest) = @_; + + return unless $use_hest; + + use HyperEstraier; + my ($index_path, $index_node_url); + + if ($use_hest =~ m#^http://#) { + $index_node_url = $use_hest; + } else { + $index_path = $TopDir . '/' . $index_path; + $index_path =~ s#//#/#g; + } + return ($index_path, $index_node_url); +} + sub getFilesHyperEstraier($) { my ($param) = @_; my $offset = $param->{'offset'} || 0; $offset *= $on_page; - die "no index_path?" unless ($index_path); + die "no index_path?" unless ($hest_index_path); use HyperEstraier; + my ($index_path, $index_node_url) = getHyperEstraier_url($hest_index_path); + # open the database - my $db = HyperEstraier::Database->new(); - $db->open($index_path, $HyperEstraier::ESTDBREADER); + my $db; + if ($index_path) { + $db = HyperEstraier::Database->new(); + $db->open($index_path, $HyperEstraier::ESTDBREADER); + } elsif ($index_node_url) { + $db ||= HyperEstraier::Node->new($index_node_url); + $db->set_auth('admin', 'admin'); + } else { + die "BUG: unimplemented"; + } # create a search condition object my $cond = HyperEstraier::Condition->new(); @@ -216,23 +314,38 @@ # $cond->set_max( $offset + $on_page ); $cond->set_options( $HyperEstraier::Condition::SURE ); - $cond->set_order( 'date NUMA' ); + $cond->set_order( getSort('search', 'est', $param->{'sort'} ) ); # get the result of search - my $result = $db->search($cond, 0); - my @res; - my $hits = $result->size; + my ($result, $hits); + + if ($index_path) { + $result = $db->search($cond, 0); + $hits = $result->size; + } elsif ($index_node_url) { + $result = $db->search($cond, 0); + $hits = $result->doc_num; + } else { + die "BUG: unimplemented"; + } # for each document in result for my $i ($offset .. ($offset + $on_page - 1)) { last if ($i >= $hits); - my $id = $result->get($i); - my $doc = $db->get_doc($id, 0); + my $doc; + if ($index_path) { + my $id = $result->get($i); + $doc = $db->get_doc($id, 0); + } elsif ($index_node_url) { + $doc = $result->get_doc($i); + } else { + die "BUG: unimplemented"; + } my $row; - foreach my $c (qw/fid hname sname backupnum fiilename filepath date type size/) { + foreach my $c (qw/fid hname sname backupnum filepath date type size/) { $row->{$c} = $doc->attr($c); } push @res, $row; @@ -241,125 +354,520 @@ return ($hits, \@res); } -sub getBackupsNotBurned() { +sub getGzipName($$$) +{ + my ($host, $share, $backupnum) = @_; + my $ret = $Conf{GzipSchema}; + + $share =~ s/\//_/g; + $ret =~ s/\\h/$host/ge; + $ret =~ s/\\s/$share/ge; + $ret =~ s/\\n/$backupnum/ge; + + $ret =~ s/__+/_/g; + + return $ret; + +} + +sub get_tgz_size_by_name($) { + my $name = shift; + + my $tgz = $Conf{InstallDir}.'/'.$Conf{GzipTempDir}.'/'.$name; + + my $size = -1; + + if (-f $tgz) { + $size = (stat($tgz))[7]; + } elsif (-d $tgz) { + opendir(my $dir, $tgz) || die "can't opendir $tgz: $!"; + my @parts = grep { !/^\./ && -f "$tgz/$_" } readdir($dir); + $size = 0; + foreach my $part (@parts) { + $size += (stat("$tgz/$part"))[7] || die "can't stat $tgz/$part: $!"; + } + closedir $dir; + } + + return $size; +} + +sub getGzipSize($$) +{ + my ($hostID, $backupNum) = @_; + my $sql; + my $dbh = get_dbh(); + + $sql = q{ + SELECT hosts.name as host, + shares.name as share, + backups.num as backupnum + FROM hosts, backups, shares + WHERE shares.id=backups.shareid AND + hosts.id =backups.hostid AND + hosts.id=? AND + backups.num=? + }; + my $sth = $dbh->prepare($sql); + $sth->execute($hostID, $backupNum); + + my $row = $sth->fetchrow_hashref(); + + return get_tgz_size_by_name( + getGzipName($row->{'host'}, $row->{share}, $row->{'backupnum'}) + ); +} + +sub getBackupsNotBurned($) { + my $param = shift; my $dbh = get_dbh(); - my $sql = q{ - SELECT - backups.hostID AS hostid, - min(hosts.name) AS host, - backups.num AS backupnum, - min(backups.type) AS type, - min(backups.date) AS date, - min(backups.size) AS size - FROM files - INNER JOIN shares ON files.shareID=shares.ID - INNER JOIN hosts ON hosts.ID = shares.hostID - INNER JOIN backups ON backups.num = files.backupnum and backups.hostID = hosts.ID AND backups.shareID = shares.ID - GROUP BY - backups.hostID, backups.num - ORDER BY min(backups.date) + + my $order = getSort('burn', 'sql', $param->{'sort'}); + +print STDERR "## sort=". ($param->{'sort'} || 'no sort param') . " burn sql order: $order\n"; + + my $sql = qq{ + SELECT + backups.hostID AS hostID, + hosts.name AS host, + shares.name AS share, + backups.num AS backupnum, + backups.type AS type, + backups.date AS date, + date_part('epoch',now()) - backups.date as age, + backups.size AS size, + backups.id AS id, + backups.inc_size AS inc_size, + backups.parts AS parts + FROM backups + INNER JOIN shares ON backups.shareID=shares.ID + INNER JOIN hosts ON backups.hostID = hosts.ID + LEFT OUTER JOIN archive_backup ON archive_backup.backup_id = backups.id + WHERE backups.inc_size > 0 AND backups.inc_deleted is false AND archive_backup.backup_id IS NULL + GROUP BY + backups.hostID, + hosts.name, + shares.name, + backups.num, + backups.shareid, + backups.id, + backups.type, + backups.date, + backups.size, + backups.inc_size, + backups.parts + ORDER BY $order }; my $sth = $dbh->prepare( $sql ); my @ret; $sth->execute(); while ( my $row = $sth->fetchrow_hashref() ) { - $row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); + $row->{'age'} = sprintf("%0.1f", ( $row->{'age'} / 86400 ) ); + #$row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); $row->{'size'} = sprintf("%0.2f", $row->{'size'} / 1024 / 1024); + + # do some cluster calculation (approximate) and convert to kB + $row->{'inc_size'} = int(($row->{'inc_size'} + 1023 ) / ( 2 * 1024 ) * 2); push @ret, $row; } return @ret; } -sub displayBackupsGrid() - { - my $retHTML = ""; - my $addForm = 1; - - if ($addForm) { +sub displayBackupsGrid($) { - $retHTML .= < + my $param = shift; + + my $retHTML .= q{ +
+ }; + + $retHTML .= <<'EOF3'; + + +
+ +Size: kB + +
+
 
+
 
+
0%
+
+
+ +
 
+ +Note: + + + + +
+ +EOF3 + $retHTML .= q{ + + + + + + } . + sort_header($param, 'Share', 'share', 'center') . + sort_header($param, '#', 'num', 'center') . + qq{ + + } . + sort_header($param, 'Date', 'date', 'center') . + sort_header($param, 'Age/days', 'age', 'center') . + sort_header($param, 'Size/Mb', 'size', 'center') . + sort_header($param, 'gzip size/Kb', 'incsize', 'center') . + qq{ + + }; + + my @color = (' bgcolor="#e0e0e0"', ''); + + my $i = 0; + my $host = ''; + + foreach my $backup ( getBackupsNotBurned($param) ) { + + if ($host ne $backup->{'host'}) { + $i++; + $host = $backup->{'host'}; + } + my $ftype = ""; + + my $checkbox_key = $backup->{'hostid'}. '_' .$backup->{'backupnum'} . '_' . $backup->{'id'}; + + $retHTML .= + ' + ' . + '' . + '' . + '' . + '' . + '' . + '' . + '' . + '' . - if ($addForm) { - $retHTML .= ""; + "\n"; } + + $retHTML .= "
+ + Type
'; + + if (($backup->{'inc_size'} || 0) > 0) { + $retHTML .= ' + '; + } + + $retHTML .= + '' . $backup->{'host'} . ':' . $backup->{'share'} . '' . $backup->{'backupnum'} . '' . $backup->{'type'} . '' . epoch_to_iso( $backup->{'date'} ) . '' . $backup->{'age'} . '' . $backup->{'size'} . '' . $backup->{'inc_size'} . + '
"; + $retHTML .= ""; return $retHTML; } @@ -405,11 +913,38 @@ - - - - - + }; + + sub sort_header($$$$) { + my ($param, $display, $name, $align) = @_; + + my ($sort_what, $sort_dir) = split(/_/,$param->{'sort'},2); + + my $old_sort = $param->{'sort'}; + + my $html = qq{'; + $param->{'sort'} = $old_sort; + + return $html; + } + + $retHTML .= + sort_header($param, 'Share', 'share', 'center') . + sort_header($param, 'Type and Name', 'path', 'center') . + sort_header($param, '#', 'num', 'center') . + sort_header($param, 'Size', 'size', 'center') . + sort_header($param, 'Date', 'date', 'center'); + + $retHTML .= qq{ }; @@ -429,6 +964,46 @@ return sprintf(qq{%s}, $action, @_); } + my $sth_archived; + my %archived_cache; + + sub check_archived($$$) { + my ($host, $share, $num) = @_; + + if (my $html = $archived_cache{"$host $share $num"}) { + return $html; + } + + $sth_archived ||= $dbh->prepare(qq{ + select + dvd_nr, note, + count(archive_burned.copy) as copies + from archive + inner join archive_burned on archive_burned.archive_id = archive.id + inner join archive_backup on archive.id = archive_backup.archive_id + inner join backups on backups.id = archive_backup.backup_id + inner join hosts on hosts.id = backups.hostid + inner join shares on shares.id = backups.shareid + where hosts.name = ? and shares.name = ? and backups.num = ? + group by dvd_nr, note + }); + + my @mediums; + + $sth_archived->execute($host, $share, $num); + while (my $row = $sth_archived->fetchrow_hashref()) { + push @mediums, '' .$row->{'dvd_nr'} . + ''; + } + + my $html = join(", ",@mediums); + $archived_cache{"$host $share $num"} = $html; + return $html; + } + my $i = $offset * $on_page; foreach $file (@{ $files }) { @@ -445,7 +1020,7 @@ qq{} . qq{} . qq{} . - qq{}; + qq{}; $retHTML .= ""; } @@ -460,20 +1035,26 @@ my $max_page = int( $results / $on_page ); my $page = 0; - sub page_link($$$) { - my ($param,$page,$display) = @_; - - $param->{'offset'} = $page; + sub page_uri($) { + my $param = shift || die "no param?"; - my $html = '' . $display . ''; + return $uri; + } + + sub page_link($$$) { + my ($param,$page,$display) = @_; + + $param->{'offset'} = $page if (defined($page)); + + my $html = '' . $display . ''; } $retHTML .= '
';
ShareType and Name#SizeDate{'sort'} = $name . '_' . $dir; + $html .= ' style="border: 1px solid #808080;"'; + } else { + $param->{'sort'} = $name . '_a'; + } + $html .= '>' . $display . 'Media
} . restore_link( $typeStr, ${EscURI( $file->{'hname'} )}, $file->{'backupnum'}, ${EscURI( $file->{'sname'})}, ${EscURI( $file->{'filepath'} )}, $file->{'backupnum'} ) . qq{} . $file->{'size'} . qq{} . epoch_to_iso( $file->{'date'} ) . qq{} . '?' . qq{} . check_archived( $file->{'hname'}, $file->{'sname'}, $file->{'backupnum'} ) . qq{