--- trunk/lib/BackupPC/SearchLib.pm 2005/08/28 10:14:48 83 +++ trunk/lib/BackupPC/SearchLib.pm 2007/05/08 12:15:26 378 @@ -8,6 +8,8 @@ use DateTime; use vars qw(%In $MyURL); use Time::HiRes qw/time/; +use XML::Writer; +use IO::File; my $on_page = 100; my $pager_pages = 10; @@ -15,24 +17,44 @@ my $dsn = $Conf{SearchDSN}; my $db_user = $Conf{SearchUser} || ''; +my $hest_node_url = $Conf{HyperEstraierIndex}; + +my $dbh; +my $Dir = $Conf{$InstallDir}."/data/log"; +open(LOG, ">>", "$Dir/LOG") +select(LOG); +$|=1; + + +sub get_dbh { + $dbh ||= DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1 } ); + return $dbh; +} + sub getUnits() { my @ret; - my $dbh = DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1 } ); - my $sth = $dbh->prepare(qq{ SELECT id, share FROM shares ORDER BY share} ); + my $dbh = get_dbh(); + my $sth = $dbh->prepare(qq{ + SELECT + shares.id as id, + hosts.name || ':' || shares.name as share + FROM shares + JOIN hosts on hostid = hosts.id + ORDER BY share + } ); $sth->execute(); push @ret, { 'id' => '', 'share' => '-'}; # dummy any while ( my $row = $sth->fetchrow_hashref() ) { push @ret, $row; } - $dbh->disconnect(); return @ret; } sub epoch_to_iso { my $t = shift || return; - my $iso = BackupPC::Lib::timeStamp($t); + my $iso = BackupPC::Lib::timeStamp(undef, $t); $iso =~ s/\s/ /g; return $iso; } @@ -43,25 +65,44 @@ sub mk_epoch_date($$) { my ($name,$suffix) = @_; - my $yyyy = $param->{ $name . '_year_' . $suffix} || return; + my $yyyy = $param->{ $name . '_year_' . $suffix} || return undef; my $mm .= $param->{ $name . '_month_' . $suffix} || ( $suffix eq 'from' ? 1 : 12); my $dd .= $param->{ $name . '_day_' . $suffix} || ( $suffix eq 'from' ? 1 : 31); + + $yyyy =~ s/\D//g; + $mm =~ s/\D//g; + $dd =~ s/\D//g; + + my $h = my $m = my $s = 0; + if ($suffix eq 'to') { + $h = 23; + $m = 59; + $s = 59; + } + my $dt = new DateTime( year => $yyyy, month => $mm, - day => $dd + day => $dd, + hour => $h, + minute => $m, + second => $s, ); + print STDERR "mk_epoch_date($name,$suffix) [$yyyy-$mm-$dd] = " . $dt->ymd . " " . $dt->hms . "\n"; return $dt->epoch || 'NULL'; } - return ( + my @ret = ( mk_epoch_date('search_backup', 'from'), mk_epoch_date('search_backup', 'to'), mk_epoch_date('search', 'from'), mk_epoch_date('search', 'to'), ); + + return @ret; + } @@ -76,61 +117,131 @@ push @conditions, qq{ files.date >= $files_from } if ($files_from); push @conditions, qq{ files.date <= $files_to } if ($files_to); - print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" | ",@conditions); + print STDERR "backup: $backup_from - $backup_to files: $files_from - $files_to cond:" . join(" and ",@conditions); push( @conditions, ' files.shareid = ' . $param->{'search_share'} ) if ($param->{'search_share'}); push (@conditions, " upper(files.path) LIKE upper('%".$param->{'search_filename'}."%')") if ($param->{'search_filename'}); + if ( $param->{burned} ) { + my $is_what = 'is null'; + $is_what = '= 1' if ($param->{burned} eq 'burned'); + push @conditions, "archive_burned.part $is_what"; + push @conditions, "archive_burned.copy $is_what"; + } + return join(" and ", @conditions); } +my $sort_def = { + search => { + default => 'date_a', + sql => { + share_d => 'shares.name DESC', + share_a => 'shares.name ASC', + path_d => 'files.path DESC', + path_a => 'files.path ASC', + num_d => 'files.backupnum DESC', + num_a => 'files.backupnum ASC', + size_d => 'files.size DESC', + size_a => 'files.size ASC', + date_d => 'files.date DESC', + date_a => 'files.date ASC', + }, + est => { + share_d => 'sname STRD', + share_a => 'sname STRA', + path_d => 'filepath STRD', + path_a => 'filepath STRA', + num_d => 'backupnum NUMD', + num_a => 'backupnum NUMA', + size_d => 'size NUMD', + size_a => 'size NUMA', + date_d => 'date NUMD', + date_a => 'date NUMA', + } + }, burn => { + default => 'date_a', + sql => { + share_d => 'host DESC, share DESC', + share_a => 'host ASC, share ASC', + num_d => 'backupnum DESC', + num_a => 'backupnum ASC', + date_d => 'date DESC', + date_a => 'date ASC', + age_d => 'age DESC', + age_a => 'age ASC', + size_d => 'size DESC', + size_a => 'size ASC', + incsize_d => 'inc_size DESC', + incsize_a => 'inc_size ASC', + } + } +}; + +sub getSort($$$) { + my ($part,$type, $sort_order) = @_; + + die "unknown part: $part" unless ($sort_def->{$part}); + die "unknown type: $type" unless ($sort_def->{$part}->{$type}); + + $sort_order ||= $sort_def->{$part}->{'default'}; + + if (my $ret = $sort_def->{$part}->{$type}->{$sort_order}) { + return $ret; + } else { + # fallback to default sort order + return $sort_def->{$part}->{$type}->{ $sort_def->{$part}->{'default'} }; + } +} + +sub getFiles($) { + my ($param) = @_; -sub getFiles($$) { - my ($param, $offset) = @_; + my $offset = $param->{'offset'} || 0; + $offset *= $on_page; - my $dbh = DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1 } ); + my $dbh = get_dbh(); my $sql_cols = qq{ files.id AS fid, hosts.name AS hname, shares.name AS sname, - shares.share AS sharename, - files.backupNum AS backupNum, - files.name AS filename, + files.backupnum AS backupnum, files.path AS filepath, files.date AS date, - files.type AS filetype, - files.size AS size, - -- dvds.name AS dvd - null AS dvd + files.type AS type, + files.size AS size }; my $sql_from = qq{ FROM files INNER JOIN shares ON files.shareID=shares.ID INNER JOIN hosts ON hosts.ID = shares.hostID - INNER JOIN backups ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID - }; - - my $sql_dvd_from = qq{ - -- LEFT JOIN dvds ON dvds.ID = files.dvdid + INNER JOIN backups ON backups.num = files.backupnum and backups.hostID = hosts.ID AND backups.shareID = files.shareID }; my $sql_where; my $where = getWhere($param); $sql_where = " WHERE ". $where if ($where); + # do we have to add tables for burned media? + if ( $param->{burned} ) { + $sql_from .= qq{ + LEFT OUTER JOIN archive_backup on archive_backup.backup_id = backups.id + LEFT OUTER JOIN archive_burned on archive_burned.archive_id = archive_backup.archive_id + }; + } + + my $order = getSort('search', 'sql', $param->{'sort'}); + my $sql_order = qq{ - ORDER BY files.date + ORDER BY $order LIMIT $on_page OFFSET ? }; my $sql_count = qq{ select count(files.id) $sql_from $sql_where }; - my $sql_results = qq{ select $sql_cols $sql_from $sql_dvd_from $sql_where $sql_order }; - - $offset ||= 0; - $offset = ($offset * $on_page); + my $sql_results = qq{ select $sql_cols $sql_from $sql_where $sql_order }; my $sth = $dbh->prepare($sql_count); $sth->execute(); @@ -148,154 +259,694 @@ my @ret; while (my $row = $sth->fetchrow_hashref()) { - push(@ret, { - 'hname' => $row->{'hname'}, - 'sname' => $row->{'sname'}, - 'sharename' => $row->{'sharename'}, - 'backupno' => $row->{'backupnum'}, - 'fname' => $row->{'filename'}, - 'fpath' => $row->{'filepath'}, - 'networkpath' => $row->{'networkpath'}, - 'date' => $row->{'date'}, - 'type' => $row->{'filetype'}, - 'size' => $row->{'size'}, - 'id' => $row->{'fid'}, - 'dvd' => $row->{'dvd'} - }); + push @ret, $row; } $sth->finish(); - $dbh->disconnect(); return ($results, \@ret); } -sub getBackupsNotBurned() { +sub getHyperEstraier_url($) { + my ($use_hest) = @_; + + return unless $use_hest; + + use Search::Estraier 0.04; + die "direct access to Hyper Estraier datatase is no longer supported. Please use estmaster\n" + unless ($use_hest =~ m#^http://#); + + return $use_hest; +} + +sub getFilesHyperEstraier($) { + my ($param) = @_; + + my $offset = $param->{'offset'} || 0; + $offset *= $on_page; + + die "no Hyper Estraier node URL?" unless ($hest_node_url); + + # open the database + my $db; + if ($hest_node_url) { + $db ||= Search::Estraier::Node->new($hest_node_url); + $db->set_auth('admin', 'admin'); + } else { + die "BUG: unimplemented"; + } + + # create a search condition object + my $cond = Search::Estraier::Condition->new(); + + my $q = $param->{'search_filename'}; + my $shareid = $param->{'search_share'}; + + if (length($q) > 0) { + # exact match + $cond->add_attr("filepath ISTRINC $q"); + + $q =~ s/(.)/$1 /g; + # set the search phrase to the search condition object + $cond->set_phrase($q); + } + + my ($backup_from, $backup_to, $files_from, $files_to) = dates_from_form($param); + + $cond->add_attr("backup_date NUMGE $backup_from") if ($backup_from); + $cond->add_attr("backup_date NUMLE $backup_to") if ($backup_to); + + $cond->add_attr("date NUMGE $files_from") if ($files_from); + $cond->add_attr("date NUMLE $files_to") if ($files_to); + + $cond->add_attr("shareid NUMEQ $shareid") if ($shareid); + + $cond->set_max( $offset + $on_page ); + $cond->set_options( 'SURE' ); + $cond->set_order( getSort('search', 'est', $param->{'sort'} ) ); + + # get the result of search + my @res; + my ($result, $hits); + + if ($hest_node_url) { + $result = $db->search($cond, 0); + if ($result) { + $hits = $result->hits; + } else { + $hits = 0; + return ($hits,[]); + } + } else { + die "BUG: unimplemented"; + } + + # for each document in result + for my $i ($offset .. ($offset + $on_page - 1)) { + last if ($i >= $result->doc_num); + + my $doc; + if ($hest_node_url) { + $doc = $result->get_doc($i); + } else { + die "BUG: unimplemented"; + } - my $dbh = DBI->connect($dsn, $db_user, "", { RaiseError => 1, AutoCommit => 1 } ); - my $sql = q{ - SELECT - backups.hostID AS hostid, - min(hosts.name) AS host, - backups.num AS backupno, - min(backups.type) AS type, - min(backups.date) AS date, - min(backups.size) AS size - FROM files - INNER JOIN shares ON files.shareID=shares.ID - INNER JOIN hosts ON hosts.ID = shares.hostID - INNER JOIN backups ON backups.num = files.backupNum and backups.hostID = hosts.ID AND backups.shareID = shares.ID - WHERE - files.dvdid IS NULL - GROUP BY - backups.hostID, backups.num - ORDER BY min(backups.date) + my $row; + foreach my $c (qw/fid hname sname backupnum filepath date type size/) { + $row->{$c} = $doc->attr($c); + } + push @res, $row; + } + + return ($hits, \@res); +} + +sub getGzipName($$$) +{ + my ($host, $share, $backupnum) = @_; + my $ret = $Conf{GzipSchema}; + + $share =~ s/\//_/g; + $ret =~ s/\\h/$host/ge; + $ret =~ s/\\s/$share/ge; + $ret =~ s/\\n/$backupnum/ge; + + $ret =~ s/__+/_/g; + + return $ret; + +} + +sub get_tgz_size_by_name($) { + my $name = shift; + + my $tgz = $Conf{InstallDir}.'/'.$Conf{GzipTempDir}.'/'.$name; + my $size = -1; + + if (-f "${tgz}.tar.gz") { + print "stating ${tgz}.tar.gz..."; + $size = (stat("${tgz}.tar.gz"))[7]; + print "size: $size\n"; + } elsif (-d $tgz) { + print "$tgz is dir, stating files in it...\n"; + opendir(my $dir, $tgz) || die "can't opendir $tgz: $!"; + my @parts = grep { !/^\./ && !/md5/ && -f "$tgz/$_" } readdir($dir); + $size = 0; + foreach my $part (@parts) { + my $currSize = (stat("$tgz/$part"))[7]; + $size += (stat("$tgz/$part"))[7] || die "can't stat $tgz/$part: $!"; + print "\t$tgz/$part: $currSize\n"; + } + print "\ttotal $size\n"; + + closedir $dir; + } else { + return -1; + } + + return $size; +} + +sub getGzipSize($$) +{ + my ($hostID, $backupNum) = @_; + my $sql; + my $dbh = get_dbh(); + + $sql = q{ + SELECT hosts.name as host, + shares.name as share, + backups.num as backupnum + FROM hosts, backups, shares + WHERE shares.id=backups.shareid AND + hosts.id =backups.hostid AND + hosts.id=? AND + backups.num=? + }; + my $sth = $dbh->prepare($sql); + $sth->execute($hostID, $backupNum); + + my $row = $sth->fetchrow_hashref(); + + return get_tgz_size_by_name( + getGzipName($row->{'host'}, $row->{share}, $row->{'backupnum'}) + ); +} + +sub getVolumes($) { + my $id = shift; + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + + my $sth = $dbh->prepare(qq{ + select + size + from backup_parts + where backup_id = ? + order by part_nr asc + }); + + $sth->execute($id); + + my $cumulative_size = 0; + my $volumes = 1; + + while(my ($size) = $sth->fetchrow_array) { + if ($cumulative_size + $size > $max_archive_size) { + $volumes++; + $cumulative_size = $size; + } else { + $cumulative_size += $size; + } + } + + return ($volumes,$cumulative_size); +} + +sub getBackupsNotBurned($) { + + my $param = shift; + my $dbh = get_dbh(); + + my $order = getSort('burn', 'sql', $param->{'sort'}); + +print STDERR "## sort=". ($param->{'sort'} || 'no sort param') . " burn sql order: $order\n"; + + my $sql = qq{ + SELECT + backups.hostID AS hostID, + hosts.name AS host, + shares.name AS share, + backups.num AS backupnum, + backups.type AS type, + backups.date AS date, + date_part('epoch',now()) - backups.date as age, + backups.size AS size, + backups.id AS id, + backups.inc_size AS inc_size, + backups.parts AS parts + FROM backups + INNER JOIN shares ON backups.shareID=shares.ID + INNER JOIN hosts ON backups.hostID = hosts.ID + LEFT OUTER JOIN archive_backup ON archive_backup.backup_id = backups.id + WHERE backups.inc_size > 0 AND backups.size > 0 AND backups.inc_deleted is false AND archive_backup.backup_id IS NULL AND backups.parts > 0 + GROUP BY + backups.hostID, + hosts.name, + shares.name, + backups.num, + backups.shareid, + backups.id, + backups.type, + backups.date, + backups.size, + backups.inc_size, + backups.parts + ORDER BY $order }; my $sth = $dbh->prepare( $sql ); my @ret; $sth->execute(); while ( my $row = $sth->fetchrow_hashref() ) { - $row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); - $row->{'size'} = sprintf("%0.2f", $row->{'size'} / 1024 / 1024); + $row->{'age'} = sprintf("%0.1f", ( $row->{'age'} / 86400 ) ); + #$row->{'age'} = sprintf("%0.1f", ( (time() - $row->{'date'}) / 86400 ) ); + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + if ($row->{size} > $max_archive_size) { + ($row->{volumes}, $row->{inc_size_calc}) = getVolumes($row->{id}); + } + + $row->{size} = sprintf("%0.2f", $row->{size} / 1024 / 1024); + + # do some cluster calculation (approximate) + $row->{inc_size} = int(( ($row->{inc_size} + 1023 ) / 2 ) * 2); + $row->{inc_size_calc} ||= $row->{inc_size}; push @ret, $row; } - return @ret; + return @ret; } -sub displayBackupsGrid() - { - my $retHTML = ""; - my $addForm = 1; - - if ($addForm) { +sub displayBackupsGrid($) { + + my $param = shift; + + my $max_archive_size = $Conf{MaxArchiveSize} || die "no MaxArchiveSize"; + my $max_archive_file_size = $Conf{MaxArchiveFileSize} || die "no MaxFileInSize"; + + my $retHTML .= q{ +
"; +function dec2hex(d) { + var hch = '0123456789ABCDEF'; + var a = d % 16; + var q = (d - a) / 16; + return hch.charAt(q) + hch.charAt(a); +} + +function pbar_set(amount, max) { + debug('pbar_set('+amount+', '+max+')'); + + if (_pbar_width == null) { + var _mc = element_id("mContainer"); + if (_pbar_width == null) _pbar_width = parseInt(_mc.offsetWidth ? (_mc.offsetWidth - 2) : 0) || null; + if (_pbar_width == null) _pbar_width = parseInt(_mc.clientWidth ? (_mc.clientWidth + 2) : 0) || null; + if (_pbar_width == null) _pbar_width = 0; + } + + var pcnt = Math.floor(amount * 100 / max); + var p90 = 100 - _pbar_warn; + var pcol = pcnt - p90; + if (Math.round(pcnt) <= 100) { + if (pcol < 0) pcol = 0; + var e = element_id("submitBurner"); + debug('enable_button'); + e.disabled = false; + var a = e.getAttributeNode('disabled') || null; + if (a) e.removeAttributeNode(a); + } else { + debug('disable button'); + pcol = _pbar_warn; + var e = element_id("submitBurner"); + if (!e.disabled) e.disabled = true; } + var col_g = Math.floor((_pbar_warn - pcol) * 255 / _pbar_warn); + var col = '#FF' + dec2hex(col_g) + '00'; + + //debug('pcol: '+pcol+' g:'+col_g+' _pbar_warn:'+ _pbar_warn + ' color: '+col); + element_id("gradient").style.backgroundColor = col; + + element_id("progressIndicator").innerHTML = pcnt + '%'; + //element_id("progressIndicator").innerHTML = amount; + + element_id("mask").style.clip = 'rect(' + Array( + '0px', + element_id("mask").offsetWidth + 'px', + element_id("mask").offsetHeight + 'px', + Math.round(_pbar_width * amount / max) + 'px' + ).join(' ') + ')'; +} + +if (!self.body) self.body = new Object(); +self.onload = self.document.onload = self.body.onload = function() { + //pbar_reset(); + sumiraj(); +}; + +// --> + ++ + | + } . + sort_header($param, 'Share', 'share', 'center') . + sort_header($param, '#', 'num', 'center') . + qq{ +Type | + } . + sort_header($param, 'Date', 'date', 'center') . + sort_header($param, 'Age/days', 'age', 'center') . + sort_header($param, 'Size/Mb', 'size', 'center') . + sort_header($param, 'gzip size/Kb', 'incsize', 'center') . + qq{ +medias | ||||||
'; + + if (($backup->{'inc_size'} || 0) > 0) { + $retHTML .= ' + '; + } + + my $img_url = $Conf{CgiImageDirURL}; + + $retHTML .= + ' | ' . + '' . $backup->{'host'} . ':' . $backup->{'share'} . ' | ' . + '' . $backup->{'backupnum'} . ' | ' . + '' . $backup->{'type'} . ' | ' . + '' . epoch_to_iso( $backup->{'date'} ) . ' | ' . + '' . $backup->{'age'} . ' | ' . + '' . $backup->{'size'} . ' | ' . + '' . sprintf("%0.1f", $backup->{'inc_size'} / 1024 ) . + ' | ' . + '' . + '' . ( qq{![]() | ' .
+
+ "