Annotation of /trunk/bin/BackupPC_updatedb

#!/usr/local/bin/perl -w

use strict;
use lib "__INSTALLDIR__/lib";

use DBI;
use BackupPC::Lib;
use BackupPC::View;
use Data::Dumper;
use Getopt::Std;
use Time::HiRes qw/time/;
use File::Pid;
use POSIX qw/strftime/;

use constant BPC_FTYPE_DIR => 5;

my $debug = 0;
$|=1;

my $start_t = time();

my $pidfile = new File::Pid;

if (my $pid = $pidfile->running ) {
        die "$0 already running: $pid\n";
} elsif ($pidfile->pid ne $$) {
        $pidfile->remove;
        $pidfile = new File::Pid;
}
$pidfile->write;
print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";

my $t_fmt = '%Y-%m-%d %H:%M:%S';

my $hosts;
my $bpc = BackupPC::Lib->new || die;
my %Conf = $bpc->Conf();
my $TopDir = $bpc->TopDir();
my $beenThere = {};

my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
my $user = $Conf{SearchUser} || '';

my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });

my %opt;

if ( !getopts("cdm:v:", \%opt ) ) {
        print STDERR <<EOF;
usage: $0 [-c|-d] [-m num] [-v|-v level]

Options:
        -c      create database on first use
        -d      delete database before import
        -m num  import just num increments for one host
        -v num  set verbosity (debug) level (default $debug)
EOF
        exit 1;
}

###################################create tables############################3

if ($opt{c}) {
        sub do_index {
                my $index = shift || return;
                my ($table,$col,$unique) = split(/_/, $index);
                $unique ||= '';
                $index =~ s/,/_/g;
                $dbh->do(qq{ create $unique index $index on $table($col) });
        }

        print "creating tables...\n";
      
        $dbh->do(qq{
                create table hosts (
                        ID      SERIAL          PRIMARY KEY,
                        name    VARCHAR(30)     NOT NULL,
                        IP      VARCHAR(15)
                );            
        });
              
        $dbh->do(qq{
                create table shares (
                        ID      SERIAL          PRIMARY KEY,
                        hostID  INTEGER         NOT NULL references hosts(id),
                        name    VARCHAR(30)     NOT NULL,
                        share   VARCHAR(200)    NOT NULL,
                        localpath VARCHAR(200)      
                );            
        });
        
        $dbh->do(qq{
                create table backups (
                        hostID  INTEGER         NOT NULL references hosts(id),
                        num     INTEGER         NOT NULL,
                        date    integer         NOT NULL, 
                        type    CHAR(4)         not null,
                        shareID integer         not null references shares(id),
                        size    integer         not null,
                        PRIMARY KEY(hostID, num, shareID) 
                );            
        });

        #do_index('backups_hostid,num_unique');

        $dbh->do(qq{
                create table dvds (
                        ID      SERIAL          PRIMARY KEY, 
                        num     INTEGER         NOT NULL,
                        name    VARCHAR(255)    NOT NULL,
                        mjesto  VARCHAR(255)
                );
        });

        $dbh->do(qq{     
                create table files (
                        ID      SERIAL          PRIMARY KEY,  
                        shareID INTEGER         NOT NULL references shares(id),
                        backupNum  INTEGER      NOT NULL,
                        name       VARCHAR(255) NOT NULL,
                        path       VARCHAR(255) NOT NULL,
                        date       integer      NOT NULL,
                        type       INTEGER      NOT NULL,
                        size       INTEGER      NOT NULL,
                        dvdid      INTEGER      references dvds(id)     
                );
        });

        print "creating indexes:";

        foreach my $index (qw(
                hosts_name
                backups_hostID
                backups_num
                shares_hostID
                shares_name
                files_shareID
                files_path
                files_name
                files_date
                files_size
        )) {
                print " $index";
                do_index($index);
        }
        print "...\n";

        $dbh->commit;

}

if ($opt{d}) {
        print "deleting ";
        foreach my $table (qw(files dvds backups shares hosts)) {
                print "$table ";
                $dbh->do(qq{ DELETE FROM $table });
        }
        print " done...\n";

        $dbh->commit;
}

if ($opt{v}) {
        print "Debug level at $opt{v}\n";
        $debug = $opt{v};
}

#################################INSERT VALUES#############################

# get hosts
$hosts = $bpc->HostInfoRead();
my $hostID;
my $shareID;

my $sth;

$sth->{insert_hosts} = $dbh->prepare(qq{
INSERT INTO hosts (name, IP) VALUES (?,?)
});

$sth->{hosts_by_name} = $dbh->prepare(qq{
SELECT ID FROM hosts WHERE name=?
});

$sth->{backups_count} = $dbh->prepare(qq{
SELECT COUNT(*)
FROM backups
WHERE hostID=? AND num=? AND shareid=?
});

$sth->{insert_backups} = $dbh->prepare(qq{
INSERT INTO backups (hostID, num, date, type, shareid, size)
VALUES (?,?,?,?,?,?)
});

$sth->{insert_files} = $dbh->prepare(qq{
INSERT INTO files
        (shareID, backupNum, name, path, date, type, size)
        VALUES (?,?,?,?,?,?,?)
});

sub fmt_time {
        my $t = shift || return;
        my $out = "";
        my ($ss,$mm,$hh) = gmtime($t);
        $out .= "${hh}h" if ($hh);
        $out .= sprintf("%02d:%02d", $mm,$ss);
        return $out;
}

foreach my $host_key (keys %{$hosts}) {

        my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key";

        $sth->{hosts_by_name}->execute($hosts->{$host_key}->{'host'});

        unless (($hostID) = $sth->{hosts_by_name}->fetchrow_array()) {
                $sth->{insert_hosts}->execute(
                        $hosts->{$host_key}->{'host'},
                        $hosts->{$host_key}->{'ip'}
                );

                $hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
        }

        print "host ".$hosts->{$host_key}->{'host'}.": ";
 
        # get backups for a host
        my @backups = $bpc->BackupInfoRead($hostname);
        my $incs = scalar @backups;
        print  "$incs increments\n";

        my $inc_nr = 0;
        $beenThere = {};

        foreach my $backup (@backups) {

                $inc_nr++;
                last if ($opt{m} && $inc_nr > $opt{m});

                my $backupNum = $backup->{'num'};
                my @backupShares = ();

                printf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n", 
                        $hosts->{$host_key}->{'host'},
                        $inc_nr, $incs, $backupNum, 
                        $backup->{type} || '?',
                        $backup->{nFilesNew} || '?', $backup->{nFiles} || '?',
                        strftime($t_fmt,localtime($backup->{startTime})),
                        fmt_time($backup->{endTime} - $backup->{startTime})
                );

                my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
                foreach my $share ($files->shareList($backupNum)) {

                        my $t = time();

                        $shareID = getShareID($share, $hostID, $hostname);
                
                        $sth->{backups_count}->execute($hostID, $backupNum, $shareID);
                        my ($count) = $sth->{backups_count}->fetchrow_array();
                        # skip if allready in database!
                        next if ($count > 0);

                        # dump some log
                        print strftime($t_fmt,localtime())," ", $share;

                        my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);

                        $sth->{insert_backups}->execute(
                                $hostID,
                                $backupNum,
                                $backup->{'endTime'},
                                $backup->{'type'},
                                $shareID,
                                $size,
                        );

                        print " commit";
                        $dbh->commit();

                        my $dur = (time() - $t) || 1;
                        printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n",
                                $nf, $f, $nd, $d,
                                ($size / 1024 / 1024),
                                ( ($f+$d) / $dur ),
                                fmt_time($dur)
                        );
                }

        }
}
undef $sth;
$dbh->commit();
$dbh->disconnect();

print "total duration: ",fmt_time(time() - $start_t),"\n";

$pidfile->remove;

sub getShareID() {

        my ($share, $hostID, $hostname) = @_;

        $sth->{share_id} ||= $dbh->prepare(qq{
                SELECT ID FROM shares WHERE hostID=? AND name=?
        });

        $sth->{share_id}->execute($hostID,$share);

        my ($id) = $sth->{share_id}->fetchrow_array();

        return $id if (defined($id));

        $sth->{insert_share} ||= $dbh->prepare(qq{
                INSERT INTO shares 
                        (hostID,name,share,localpath) 
                VALUES (?,?,?,?)
        });

        my $drop_down = $hostname . '/' . $share;
        $drop_down =~ s#//+#/#g;

        $sth->{insert_share}->execute($hostID,$share, $drop_down ,undef);
        return $dbh->last_insert_id(undef,undef,'shares',undef);
}

sub found_in_db {

        my @data = @_;
        shift @data;

        my ($key, $shareID,undef,$name,$path,$date,undef,$size) = @_;

        return $beenThere->{$key} if (defined($beenThere->{$key}));

        $sth->{file_in_db} ||= $dbh->prepare(qq{
                SELECT 1 FROM files
                WHERE shareID = ? and
                        path = ? and 
                        date = ? and
                        size = ?
                LIMIT 1
        });

        my @param = ($shareID,$path,$date,$size);
        $sth->{file_in_db}->execute(@param);
        my $rows = $sth->{file_in_db}->rows;
        print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);

        $beenThere->{$key}++;

        $sth->{'insert_files'}->execute(@data) unless ($rows);
        return $rows;
}

####################################################
# recursing through filesystem structure and       #
# and returning flattened files list               #
####################################################
sub recurseDir($$$$$$$$) {

        my ($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID) = @_;

        print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);

        my ($nr_files, $new_files, $nr_dirs, $new_dirs, $size) = (0,0,0,0,0);

        { # scope
                my @stack;

                print STDERR "# dirAttrib($backupNum, $share, $dir)\n" if ($debug >= 2);
                my $filesInBackup = $files->dirAttrib($backupNum, $share, $dir);

                # first, add all the entries in current directory
                foreach my $path_key (keys %{$filesInBackup}) {
                        print STDERR "# file ",Dumper($filesInBackup->{$path_key}),"\n" if ($debug >= 3);
                        my @data = (
                                $shareID,
                                $backupNum,
                                $path_key,
                                $filesInBackup->{$path_key}->{'relPath'},
                                $filesInBackup->{$path_key}->{'mtime'},
                                $filesInBackup->{$path_key}->{'type'},
                                $filesInBackup->{$path_key}->{'size'}
                        );

                        my $key = join(" ", (
                                $shareID,
                                $dir,
                                $path_key,
                                $filesInBackup->{$path_key}->{'mtime'},
                                $filesInBackup->{$path_key}->{'size'}
                        ));

                        my $found;
                        if (! defined($beenThere->{$key}) && ! ($found = found_in_db($key, @data)) ) {
                                print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);

                                if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
                                        $new_dirs++ unless ($found);
                                        print STDERR " dir\n" if ($debug >= 2);
                                } else {
                                        $new_files++ unless ($found);
                                        print STDERR " file\n" if ($debug >= 2);
                                }
                                $size += $filesInBackup->{$path_key}->{'size'} || 0;
                        }

                        if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
                                $nr_dirs++;

                                my $full_path = $dir . '/' . $path_key;
                                push @stack, $full_path;
                                print STDERR "### store to stack: $full_path\n" if ($debug >= 3);

#                               my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $backups, $backupNum, $share, $path_key, $shareID) unless ($beenThere->{$key});
#
#                               $nr_files += $f;
#                               $new_files += $nf;
#                               $nr_dirs += $d;
#                               $new_dirs += $nd;

                        } else {
                                $nr_files++;
                        }
                }

                print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);

                while ( my $dir = shift @stack ) {
                        my ($f,$nf,$d,$nd, $s) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
                        print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
                        $nr_files += $f;
                        $new_files += $nf;
                        $nr_dirs += $d;
                        $new_dirs += $nd;
                        $size += $s;
                }
        }

        return ($nr_files, $new_files, $nr_dirs, $new_dirs, $size);
}

1	dpavlin	14	#!/usr/local/bin/perl -w
2	dpavlin	6
3			use strict;
4	dpavlin	48	use lib "__INSTALLDIR__/lib";
5
6	dpavlin	6	use DBI;
7			use BackupPC::Lib;
8			use BackupPC::View;
9			use Data::Dumper;
10			use Getopt::Std;
11	dpavlin	37	use Time::HiRes qw/time/;
12	dpavlin	38	use File::Pid;
13	dpavlin	37	use POSIX qw/strftime/;
14	dpavlin	48
15	dpavlin	6	use constant BPC_FTYPE_DIR => 5;
16
17	dpavlin	30	my $debug = 0;
18	dpavlin	14	$\|=1;
19	dpavlin	6
20	dpavlin	51	my $start_t = time();
21
22	dpavlin	38	my $pidfile = new File::Pid;
23
24			if (my $pid = $pidfile->running ) {
25			die "$0 already running: $pid\n";
26			} elsif ($pidfile->pid ne $$) {
27			$pidfile->remove;
28			$pidfile = new File::Pid;
29			}
30	dpavlin	39	$pidfile->write;
31			print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";
32	dpavlin	38
33	dpavlin	37	my $t_fmt = '%Y-%m-%d %H:%M:%S';
34
35	dpavlin	6	my $hosts;
36			my $bpc = BackupPC::Lib->new \|\| die;
37			my %Conf = $bpc->Conf();
38			my $TopDir = $bpc->TopDir();
39	dpavlin	14	my $beenThere = {};
40	dpavlin	6
41	dpavlin	51	my $dsn = $Conf{SearchDSN} \|\| die "Need SearchDSN in config.pl\n";
42			my $user = $Conf{SearchUser} \|\| '';
43	dpavlin	6
44	dpavlin	49	my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
45
46	dpavlin	6	my %opt;
47
48	dpavlin	44	if ( !getopts("cdm:v:", \%opt ) ) {
49	dpavlin	6	print STDERR <<EOF;
50	dpavlin	44	usage: $0 [-c\|-d] [-m num] [-v\|-v level]
51	dpavlin	6
52			Options:
53	dpavlin	14	-c create database on first use
54			-d delete database before import
55			-m num import just num increments for one host
56	dpavlin	44	-v num set verbosity (debug) level (default $debug)
57	dpavlin	6	EOF
58			exit 1;
59			}
60
61			###################################create tables############################3
62
63	dpavlin	14	if ($opt{c}) {
64	dpavlin	49	sub do_index {
65			my $index = shift \|\| return;
66			my ($table,$col,$unique) = split(/_/, $index);
67			$unique \|\|= '';
68	dpavlin	52	$index =~ s/,/_/g;
69	dpavlin	49	$dbh->do(qq{ create $unique index $index on $table($col) });
70			}
71
72	dpavlin	14	print "creating tables...\n";
73	dpavlin	6
74	dpavlin	14	$dbh->do(qq{
75			create table hosts (
76	dpavlin	49	ID SERIAL PRIMARY KEY,
77	dpavlin	14	name VARCHAR(30) NOT NULL,
78			IP VARCHAR(15)
79	dpavlin	6	);
80	dpavlin	14	});
81	dpavlin	6
82	dpavlin	14	$dbh->do(qq{
83			create table shares (
84	dpavlin	49	ID SERIAL PRIMARY KEY,
85	dpavlin	14	hostID INTEGER NOT NULL references hosts(id),
86			name VARCHAR(30) NOT NULL,
87			share VARCHAR(200) NOT NULL,
88			localpath VARCHAR(200)
89	dpavlin	6	);
90	dpavlin	14	});
91	dpavlin	6
92	dpavlin	14	$dbh->do(qq{
93			create table backups (
94			hostID INTEGER NOT NULL references hosts(id),
95			num INTEGER NOT NULL,
96	dpavlin	49	date integer NOT NULL,
97			type CHAR(4) not null,
98	dpavlin	65	shareID integer not null references shares(id),
99	dpavlin	66	size integer not null,
100	dpavlin	65	PRIMARY KEY(hostID, num, shareID)
101	dpavlin	6	);
102	dpavlin	14	});
103	dpavlin	6
104	dpavlin	65	#do_index('backups_hostid,num_unique');
105	dpavlin	49
106	dpavlin	14	$dbh->do(qq{
107			create table dvds (
108	dpavlin	49	ID SERIAL PRIMARY KEY,
109	dpavlin	14	num INTEGER NOT NULL,
110			name VARCHAR(255) NOT NULL,
111			mjesto VARCHAR(255)
112	dpavlin	6	);
113	dpavlin	14	});
114	dpavlin	6
115	dpavlin	14	$dbh->do(qq{
116			create table files (
117	dpavlin	49	ID SERIAL PRIMARY KEY,
118	dpavlin	14	shareID INTEGER NOT NULL references shares(id),
119	dpavlin	52	backupNum INTEGER NOT NULL,
120	dpavlin	14	name VARCHAR(255) NOT NULL,
121			path VARCHAR(255) NOT NULL,
122	dpavlin	49	date integer NOT NULL,
123	dpavlin	14	type INTEGER NOT NULL,
124			size INTEGER NOT NULL,
125			dvdid INTEGER references dvds(id)
126	dpavlin	6	);
127	dpavlin	14	});
128	dpavlin	6
129	dpavlin	49	print "creating indexes:";
130	dpavlin	6
131	dpavlin	14	foreach my $index (qw(
132			hosts_name
133			backups_hostID
134			backups_num
135			shares_hostID
136			shares_name
137			files_shareID
138			files_path
139			files_name
140			files_date
141			files_size
142			)) {
143	dpavlin	49	print " $index";
144			do_index($index);
145	dpavlin	14	}
146	dpavlin	49	print "...\n";
147	dpavlin	14
148	dpavlin	49	$dbh->commit;
149	dpavlin	14
150			}
151
152			if ($opt{d}) {
153			print "deleting ";
154	dpavlin	49	foreach my $table (qw(files dvds backups shares hosts)) {
155	dpavlin	14	print "$table ";
156			$dbh->do(qq{ DELETE FROM $table });
157			}
158			print " done...\n";
159	dpavlin	49
160	dpavlin	51	$dbh->commit;
161	dpavlin	14	}
162
163	dpavlin	32	if ($opt{v}) {
164			print "Debug level at $opt{v}\n";
165			$debug = $opt{v};
166			}
167
168	dpavlin	6	#################################INSERT VALUES#############################
169
170			# get hosts
171	dpavlin	8	$hosts = $bpc->HostInfoRead();
172	dpavlin	6	my $hostID;
173			my $shareID;
174
175	dpavlin	14	my $sth;
176	dpavlin	6
177	dpavlin	14	$sth->{insert_hosts} = $dbh->prepare(qq{
178			INSERT INTO hosts (name, IP) VALUES (?,?)
179			});
180	dpavlin	6
181	dpavlin	14	$sth->{hosts_by_name} = $dbh->prepare(qq{
182			SELECT ID FROM hosts WHERE name=?
183			});
184
185	dpavlin	65	$sth->{backups_count} = $dbh->prepare(qq{
186	dpavlin	14	SELECT COUNT(*)
187			FROM backups
188	dpavlin	65	WHERE hostID=? AND num=? AND shareid=?
189	dpavlin	14	});
190
191			$sth->{insert_backups} = $dbh->prepare(qq{
192	dpavlin	66	INSERT INTO backups (hostID, num, date, type, shareid, size)
193			VALUES (?,?,?,?,?,?)
194	dpavlin	14	});
195
196			$sth->{insert_files} = $dbh->prepare(qq{
197			INSERT INTO files
198	dpavlin	62	(shareID, backupNum, name, path, date, type, size)
199			VALUES (?,?,?,?,?,?,?)
200	dpavlin	14	});
201
202	dpavlin	50	sub fmt_time {
203			my $t = shift \|\| return;
204			my $out = "";
205			my ($ss,$mm,$hh) = gmtime($t);
206			$out .= "${hh}h" if ($hh);
207			$out .= sprintf("%02d:%02d", $mm,$ss);
208			return $out;
209			}
210
211	dpavlin	14	foreach my $host_key (keys %{$hosts}) {
212
213			my $hostname = $hosts->{$host_key}->{'host'} \|\| die "can't find host for $host_key";
214
215			$sth->{hosts_by_name}->execute($hosts->{$host_key}->{'host'});
216
217			unless (($hostID) = $sth->{hosts_by_name}->fetchrow_array()) {
218			$sth->{insert_hosts}->execute(
219			$hosts->{$host_key}->{'host'},
220			$hosts->{$host_key}->{'ip'}
221			);
222
223	dpavlin	49	$hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
224	dpavlin	14	}
225
226	dpavlin	67	print "host ".$hosts->{$host_key}->{'host'}.": ";
227	dpavlin	6
228	dpavlin	14	# get backups for a host
229			my @backups = $bpc->BackupInfoRead($hostname);
230	dpavlin	67	my $incs = scalar @backups;
231			print "$incs increments\n";
232	dpavlin	6
233	dpavlin	14	my $inc_nr = 0;
234	dpavlin	67	$beenThere = {};
235	dpavlin	14
236			foreach my $backup (@backups) {
237	dpavlin	40
238	dpavlin	14	$inc_nr++;
239			last if ($opt{m} && $inc_nr > $opt{m});
240
241			my $backupNum = $backup->{'num'};
242			my @backupShares = ();
243
244	dpavlin	67	printf("%-10s %2d/%-2d #%-2d %s %5s/%5s files (date: %s dur: %s)\n",
245			$hosts->{$host_key}->{'host'},
246			$inc_nr, $incs, $backupNum,
247			$backup->{type} \|\| '?',
248			$backup->{nFilesNew} \|\| '?', $backup->{nFiles} \|\| '?',
249	dpavlin	57	strftime($t_fmt,localtime($backup->{startTime})),
250	dpavlin	67	fmt_time($backup->{endTime} - $backup->{startTime})
251			);
252	dpavlin	14
253	dpavlin	34	my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
254	dpavlin	14	foreach my $share ($files->shareList($backupNum)) {
255
256	dpavlin	37	my $t = time();
257
258	dpavlin	14	$shareID = getShareID($share, $hostID, $hostname);
259
260	dpavlin	65	$sth->{backups_count}->execute($hostID, $backupNum, $shareID);
261			my ($count) = $sth->{backups_count}->fetchrow_array();
262			# skip if allready in database!
263			next if ($count > 0);
264
265			# dump some log
266			print strftime($t_fmt,localtime())," ", $share;
267
268	dpavlin	66	my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
269	dpavlin	65
270			$sth->{insert_backups}->execute(
271			$hostID,
272			$backupNum,
273			$backup->{'endTime'},
274			$backup->{'type'},
275	dpavlin	66	$shareID,
276			$size,
277	dpavlin	65	);
278
279			print " commit";
280			$dbh->commit();
281
282	dpavlin	50	my $dur = (time() - $t) \|\| 1;
283	dpavlin	66	printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n",
284	dpavlin	37	$nf, $f, $nd, $d,
285	dpavlin	66	($size / 1024 / 1024),
286	dpavlin	50	( ($f+$d) / $dur ),
287			fmt_time($dur)
288	dpavlin	37	);
289	dpavlin	14	}
290	dpavlin	29
291	dpavlin	6	}
292			}
293	dpavlin	14	undef $sth;
294	dpavlin	6	$dbh->commit();
295			$dbh->disconnect();
296
297	dpavlin	51	print "total duration: ",fmt_time(time() - $start_t),"\n";
298
299	dpavlin	38	$pidfile->remove;
300
301	dpavlin	14	sub getShareID() {
302	dpavlin	6
303	dpavlin	14	my ($share, $hostID, $hostname) = @_;
304
305			$sth->{share_id} \|\|= $dbh->prepare(qq{
306			SELECT ID FROM shares WHERE hostID=? AND name=?
307			});
308
309			$sth->{share_id}->execute($hostID,$share);
310
311			my ($id) = $sth->{share_id}->fetchrow_array();
312
313			return $id if (defined($id));
314
315			$sth->{insert_share} \|\|= $dbh->prepare(qq{
316			INSERT INTO shares
317			(hostID,name,share,localpath)
318			VALUES (?,?,?,?)
319			});
320
321	dpavlin	25	my $drop_down = $hostname . '/' . $share;
322			$drop_down =~ s#//+#/#g;
323
324			$sth->{insert_share}->execute($hostID,$share, $drop_down ,undef);
325	dpavlin	49	return $dbh->last_insert_id(undef,undef,'shares',undef);
326	dpavlin	6	}
327
328	dpavlin	14	sub found_in_db {
329
330	dpavlin	48	my @data = @_;
331			shift @data;
332	dpavlin	14
333	dpavlin	74	my ($key, $shareID,undef,$name,$path,$date,undef,$size) = @_;
334	dpavlin	48
335			return $beenThere->{$key} if (defined($beenThere->{$key}));
336
337	dpavlin	14	$sth->{file_in_db} \|\|= $dbh->prepare(qq{
338	dpavlin	48	SELECT 1 FROM files
339	dpavlin	14	WHERE shareID = ? and
340			path = ? and
341			date = ? and
342			size = ?
343	dpavlin	74	LIMIT 1
344	dpavlin	14	});
345
346	dpavlin	67	my @param = ($shareID,$path,$date,$size);
347	dpavlin	14	$sth->{file_in_db}->execute(@param);
348	dpavlin	48	my $rows = $sth->{file_in_db}->rows;
349	dpavlin	74	print STDERR "## found_in_db($shareID,$path,$date,$size) ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
350	dpavlin	48
351			$beenThere->{$key}++;
352
353			$sth->{'insert_files'}->execute(@data) unless ($rows);
354	dpavlin	14	return $rows;
355	dpavlin	6	}
356
357			####################################################
358			# recursing through filesystem structure and #
359			# and returning flattened files list #
360			####################################################
361	dpavlin	14	sub recurseDir($$$$$$$$) {
362	dpavlin	6
363	dpavlin	35	my ($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID) = @_;
364	dpavlin	14
365	dpavlin	44	print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);
366	dpavlin	29
367	dpavlin	66	my ($nr_files, $new_files, $nr_dirs, $new_dirs, $size) = (0,0,0,0,0);
368	dpavlin	14
369	dpavlin	27	{ # scope
370	dpavlin	29	my @stack;
371	dpavlin	14
372	dpavlin	45	print STDERR "# dirAttrib($backupNum, $share, $dir)\n" if ($debug >= 2);
373	dpavlin	27	my $filesInBackup = $files->dirAttrib($backupNum, $share, $dir);
374	dpavlin	14
375	dpavlin	27	# first, add all the entries in current directory
376			foreach my $path_key (keys %{$filesInBackup}) {
377	dpavlin	66	print STDERR "# file ",Dumper($filesInBackup->{$path_key}),"\n" if ($debug >= 3);
378	dpavlin	27	my @data = (
379			$shareID,
380			$backupNum,
381			$path_key,
382			$filesInBackup->{$path_key}->{'relPath'},
383			$filesInBackup->{$path_key}->{'mtime'},
384			$filesInBackup->{$path_key}->{'type'},
385			$filesInBackup->{$path_key}->{'size'}
386			);
387
388			my $key = join(" ", (
389			$shareID,
390			$dir,
391			$path_key,
392			$filesInBackup->{$path_key}->{'mtime'},
393			$filesInBackup->{$path_key}->{'size'}
394			));
395
396	dpavlin	70	my $found;
397			if (! defined($beenThere->{$key}) && ! ($found = found_in_db($key, @data)) ) {
398	dpavlin	30	print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
399	dpavlin	48
400	dpavlin	27	if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
401	dpavlin	70	$new_dirs++ unless ($found);
402	dpavlin	30	print STDERR " dir\n" if ($debug >= 2);
403	dpavlin	27	} else {
404	dpavlin	70	$new_files++ unless ($found);
405	dpavlin	30	print STDERR " file\n" if ($debug >= 2);
406	dpavlin	27	}
407	dpavlin	66	$size += $filesInBackup->{$path_key}->{'size'} \|\| 0;
408	dpavlin	27	}
409
410	dpavlin	14	if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
411	dpavlin	27	$nr_dirs++;
412
413	dpavlin	29	my $full_path = $dir . '/' . $path_key;
414			push @stack, $full_path;
415	dpavlin	30	print STDERR "### store to stack: $full_path\n" if ($debug >= 3);
416	dpavlin	29
417	dpavlin	27	# my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $backups, $backupNum, $share, $path_key, $shareID) unless ($beenThere->{$key});
418			#
419			# $nr_files += $f;
420			# $new_files += $nf;
421			# $nr_dirs += $d;
422			# $new_dirs += $nd;
423
424	dpavlin	14	} else {
425	dpavlin	27	$nr_files++;
426	dpavlin	14	}
427			}
428
429	dpavlin	30	print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);
430	dpavlin	14
431	dpavlin	29	while ( my $dir = shift @stack ) {
432	dpavlin	66	my ($f,$nf,$d,$nd, $s) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
433	dpavlin	30	print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
434	dpavlin	29	$nr_files += $f;
435			$new_files += $nf;
436			$nr_dirs += $d;
437			$new_dirs += $nd;
438	dpavlin	66	$size += $s;
439	dpavlin	29	}
440	dpavlin	14	}
441
442	dpavlin	66	return ($nr_files, $new_files, $nr_dirs, $new_dirs, $size);
443	dpavlin	6	}
444