Annotation of /trunk/bin/BackupPC_updatedb

#!/usr/local/bin/perl -w

use strict;
use lib "__INSTALLDIR__/lib";

use DBI;
use BackupPC::Lib;
use BackupPC::View;
use Data::Dumper;
use Getopt::Std;
use Time::HiRes qw/time/;
use File::Pid;
use POSIX qw/strftime/;

use constant BPC_FTYPE_DIR => 5;

my $debug = 0;
$|=1;

my $start_t = time();

my $pidfile = new File::Pid;

if (my $pid = $pidfile->running ) {
        die "$0 already running: $pid\n";
} elsif ($pidfile->pid ne $$) {
        $pidfile->remove;
        $pidfile = new File::Pid;
}
$pidfile->write;
print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";

my $t_fmt = '%Y-%m-%d %H:%M:%S';

my $hosts;
my $bpc = BackupPC::Lib->new || die;
my %Conf = $bpc->Conf();
my $TopDir = $bpc->TopDir();
my $beenThere = {};

my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
my $user = $Conf{SearchUser} || '';

my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });

my %opt;

if ( !getopts("cdm:v:", \%opt ) ) {
        print STDERR <<EOF;
usage: $0 [-c|-d] [-m num] [-v|-v level]

Options:
        -c      create database on first use
        -d      delete database before import
        -m num  import just num increments for one host
        -v num  set verbosity (debug) level (default $debug)
EOF
        exit 1;
}

###################################create tables############################3

if ($opt{c}) {
        sub do_index {
                my $index = shift || return;
                my ($table,$col,$unique) = split(/_/, $index);
                $unique ||= '';
                $index =~ s/,/_/g;
                $dbh->do(qq{ create $unique index $index on $table($col) });
        }

        print "creating tables...\n";
      
        $dbh->do(qq{
                create table hosts (
                        ID      SERIAL          PRIMARY KEY,
                        name    VARCHAR(30)     NOT NULL,
                        IP      VARCHAR(15)
                );            
        });
              
        $dbh->do(qq{
                create table shares (
                        ID      SERIAL          PRIMARY KEY,
                        hostID  INTEGER         NOT NULL references hosts(id),
                        name    VARCHAR(30)     NOT NULL,
                        share   VARCHAR(200)    NOT NULL,
                        localpath VARCHAR(200)      
                );            
        });
        
        $dbh->do(qq{
                create table backups (
                        hostID  INTEGER         NOT NULL references hosts(id),
                        num     INTEGER         NOT NULL,
                        date    integer         NOT NULL, 
                        type    CHAR(4)         not null,
                        shareID integer         not null references shares(id),
                        size    integer         not null,
                        PRIMARY KEY(hostID, num, shareID) 
                );            
        });

        #do_index('backups_hostid,num_unique');

        $dbh->do(qq{
                create table dvds (
                        ID      SERIAL          PRIMARY KEY, 
                        num     INTEGER         NOT NULL,
                        name    VARCHAR(255)    NOT NULL,
                        mjesto  VARCHAR(255)
                );
        });

        $dbh->do(qq{     
                create table files (
                        ID      SERIAL          PRIMARY KEY,  
                        shareID INTEGER         NOT NULL references shares(id),
                        backupNum  INTEGER      NOT NULL,
                        name       VARCHAR(255) NOT NULL,
                        path       VARCHAR(255) NOT NULL,
                        date       integer      NOT NULL,
                        type       INTEGER      NOT NULL,
                        size       INTEGER      NOT NULL,
                        dvdid      INTEGER      references dvds(id)     
                );
        });

        print "creating indexes:";

        foreach my $index (qw(
                hosts_name
                backups_hostID
                backups_num
                shares_hostID
                shares_name
                files_shareID
                files_path
                files_name
                files_date
                files_size
        )) {
                print " $index";
                do_index($index);
        }
        print "...\n";

        $dbh->commit;

}

if ($opt{d}) {
        print "deleting ";
        foreach my $table (qw(files dvds backups shares hosts)) {
                print "$table ";
                $dbh->do(qq{ DELETE FROM $table });
        }
        print " done...\n";

        $dbh->commit;
}

if ($opt{v}) {
        print "Debug level at $opt{v}\n";
        $debug = $opt{v};
}

#################################INSERT VALUES#############################

# get hosts
$hosts = $bpc->HostInfoRead();
my $hostID;
my $shareID;

my $sth;

$sth->{insert_hosts} = $dbh->prepare(qq{
INSERT INTO hosts (name, IP) VALUES (?,?)
});

$sth->{hosts_by_name} = $dbh->prepare(qq{
SELECT ID FROM hosts WHERE name=?
});

$sth->{backups_count} = $dbh->prepare(qq{
SELECT COUNT(*)
FROM backups
WHERE hostID=? AND num=? AND shareid=?
});

$sth->{insert_backups} = $dbh->prepare(qq{
INSERT INTO backups (hostID, num, date, type, shareid, size)
VALUES (?,?,?,?,?,?)
});

$sth->{insert_files} = $dbh->prepare(qq{
INSERT INTO files
        (shareID, backupNum, name, path, date, type, size)
        VALUES (?,?,?,?,?,?,?)
});

sub fmt_time {
        my $t = shift || return;
        my $out = "";
        my ($ss,$mm,$hh) = gmtime($t);
        $out .= "${hh}h" if ($hh);
        $out .= sprintf("%02d:%02d", $mm,$ss);
        return $out;
}

foreach my $host_key (keys %{$hosts}) {

        my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key";

        $sth->{hosts_by_name}->execute($hosts->{$host_key}->{'host'});

        unless (($hostID) = $sth->{hosts_by_name}->fetchrow_array()) {
                $sth->{insert_hosts}->execute(
                        $hosts->{$host_key}->{'host'},
                        $hosts->{$host_key}->{'ip'}
                );

                $hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
        }

        print("host ".$hosts->{$host_key}->{'host'}.": ");
 
        # get backups for a host
        my @backups = $bpc->BackupInfoRead($hostname);
        print scalar @backups, " increments\n";

        my $inc_nr = 0;

        foreach my $backup (@backups) {

                $inc_nr++;
                last if ($opt{m} && $inc_nr > $opt{m});

                my $backupNum = $backup->{'num'};
                my @backupShares = ();

                print $hosts->{$host_key}->{'host'},
                        "\t#$backupNum\t", $backup->{type} || '?', " ",
                        $backup->{nFilesNew} || '?', "/", $backup->{nFiles} || '?',
                        " files (date: ",
                        strftime($t_fmt,localtime($backup->{startTime})),
                        " dur: ",
                        fmt_time($backup->{endTime} - $backup->{startTime}),
                        ")\n";

                my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
                foreach my $share ($files->shareList($backupNum)) {

                        my $t = time();

                        $shareID = getShareID($share, $hostID, $hostname);
                
                        $sth->{backups_count}->execute($hostID, $backupNum, $shareID);
                        my ($count) = $sth->{backups_count}->fetchrow_array();
                        # skip if allready in database!
                        next if ($count > 0);

                        # dump some log
                        print strftime($t_fmt,localtime())," ", $share;

                        my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);

                        $sth->{insert_backups}->execute(
                                $hostID,
                                $backupNum,
                                $backup->{'endTime'},
                                $backup->{'type'},
                                $shareID,
                                $size,
                        );

                        print " commit";
                        $dbh->commit();

                        my $dur = (time() - $t) || 1;
                        printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n",
                                $nf, $f, $nd, $d,
                                ($size / 1024 / 1024),
                                ( ($f+$d) / $dur ),
                                fmt_time($dur)
                        );
                }

        }
}
undef $sth;
$dbh->commit();
$dbh->disconnect();

print "total duration: ",fmt_time(time() - $start_t),"\n";

$pidfile->remove;

sub getShareID() {

        my ($share, $hostID, $hostname) = @_;

        $sth->{share_id} ||= $dbh->prepare(qq{
                SELECT ID FROM shares WHERE hostID=? AND name=?
        });

        $sth->{share_id}->execute($hostID,$share);

        my ($id) = $sth->{share_id}->fetchrow_array();

        return $id if (defined($id));

        $sth->{insert_share} ||= $dbh->prepare(qq{
                INSERT INTO shares 
                        (hostID,name,share,localpath) 
                VALUES (?,?,?,?)
        });

        my $drop_down = $hostname . '/' . $share;
        $drop_down =~ s#//+#/#g;

        $sth->{insert_share}->execute($hostID,$share, $drop_down ,undef);
        return $dbh->last_insert_id(undef,undef,'shares',undef);
}

sub found_in_db {

        my @data = @_;
        shift @data;

        my ($key, $shareID,undef,$name,$path,undef,$date,undef,$size) = @_;

        return $beenThere->{$key} if (defined($beenThere->{$key}));

        $sth->{file_in_db} ||= $dbh->prepare(qq{
                SELECT 1 FROM files
                WHERE shareID = ? and
                        path = ? and 
                        name = ? and
                        date = ? and
                        size = ?
        });

        my @param = ($shareID,$path,$name,$date,$size);
        $sth->{file_in_db}->execute(@param);
        my $rows = $sth->{file_in_db}->rows;
        print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);

        $beenThere->{$key}++;

        $sth->{'insert_files'}->execute(@data) unless ($rows);
        return $rows;
}

####################################################
# recursing through filesystem structure and       #
# and returning flattened files list               #
####################################################
sub recurseDir($$$$$$$$) {

        my ($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID) = @_;

        print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);

        my ($nr_files, $new_files, $nr_dirs, $new_dirs, $size) = (0,0,0,0,0);

        { # scope
                my @stack;

                print STDERR "# dirAttrib($backupNum, $share, $dir)\n" if ($debug >= 2);
                my $filesInBackup = $files->dirAttrib($backupNum, $share, $dir);

                # first, add all the entries in current directory
                foreach my $path_key (keys %{$filesInBackup}) {
                        print STDERR "# file ",Dumper($filesInBackup->{$path_key}),"\n" if ($debug >= 3);
                        my @data = (
                                $shareID,
                                $backupNum,
                                $path_key,
                                $filesInBackup->{$path_key}->{'relPath'},
                                $filesInBackup->{$path_key}->{'mtime'},
                                $filesInBackup->{$path_key}->{'type'},
                                $filesInBackup->{$path_key}->{'size'}
                        );

                        my $key = join(" ", (
                                $shareID,
                                $dir,
                                $path_key,
                                $filesInBackup->{$path_key}->{'mtime'},
                                $filesInBackup->{$path_key}->{'size'}
                        ));


                        if (! defined($beenThere->{$key}) && ! found_in_db($key, @data)) {
                                print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);

                                if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
                                        $new_dirs++;
                                        print STDERR " dir\n" if ($debug >= 2);
                                } else {
                                        $new_files++;
                                        print STDERR " file\n" if ($debug >= 2);
                                }
                                $size += $filesInBackup->{$path_key}->{'size'} || 0;
                        }

                        if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
                                $nr_dirs++;

                                my $full_path = $dir . '/' . $path_key;
                                push @stack, $full_path;
                                print STDERR "### store to stack: $full_path\n" if ($debug >= 3);

#                               my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $backups, $backupNum, $share, $path_key, $shareID) unless ($beenThere->{$key});
#
#                               $nr_files += $f;
#                               $new_files += $nf;
#                               $nr_dirs += $d;
#                               $new_dirs += $nd;

                        } else {
                                $nr_files++;
                        }
                }

                print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);

                while ( my $dir = shift @stack ) {
                        my ($f,$nf,$d,$nd, $s) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
                        print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
                        $nr_files += $f;
                        $new_files += $nf;
                        $nr_dirs += $d;
                        $new_dirs += $nd;
                        $size += $s;
                }
        }

        return ($nr_files, $new_files, $nr_dirs, $new_dirs, $size);
}

1	dpavlin	14	#!/usr/local/bin/perl -w
2	dpavlin	6
3			use strict;
4	dpavlin	48	use lib "__INSTALLDIR__/lib";
5
6	dpavlin	6	use DBI;
7			use BackupPC::Lib;
8			use BackupPC::View;
9			use Data::Dumper;
10			use Getopt::Std;
11	dpavlin	37	use Time::HiRes qw/time/;
12	dpavlin	38	use File::Pid;
13	dpavlin	37	use POSIX qw/strftime/;
14	dpavlin	48
15	dpavlin	6	use constant BPC_FTYPE_DIR => 5;
16
17	dpavlin	30	my $debug = 0;
18	dpavlin	14	$\|=1;
19	dpavlin	6
20	dpavlin	51	my $start_t = time();
21
22	dpavlin	38	my $pidfile = new File::Pid;
23
24			if (my $pid = $pidfile->running ) {
25			die "$0 already running: $pid\n";
26			} elsif ($pidfile->pid ne $$) {
27			$pidfile->remove;
28			$pidfile = new File::Pid;
29			}
30	dpavlin	39	$pidfile->write;
31			print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";
32	dpavlin	38
33	dpavlin	37	my $t_fmt = '%Y-%m-%d %H:%M:%S';
34
35	dpavlin	6	my $hosts;
36			my $bpc = BackupPC::Lib->new \|\| die;
37			my %Conf = $bpc->Conf();
38			my $TopDir = $bpc->TopDir();
39	dpavlin	14	my $beenThere = {};
40	dpavlin	6
41	dpavlin	51	my $dsn = $Conf{SearchDSN} \|\| die "Need SearchDSN in config.pl\n";
42			my $user = $Conf{SearchUser} \|\| '';
43	dpavlin	6
44	dpavlin	49	my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
45
46	dpavlin	6	my %opt;
47
48	dpavlin	44	if ( !getopts("cdm:v:", \%opt ) ) {
49	dpavlin	6	print STDERR <<EOF;
50	dpavlin	44	usage: $0 [-c\|-d] [-m num] [-v\|-v level]
51	dpavlin	6
52			Options:
53	dpavlin	14	-c create database on first use
54			-d delete database before import
55			-m num import just num increments for one host
56	dpavlin	44	-v num set verbosity (debug) level (default $debug)
57	dpavlin	6	EOF
58			exit 1;
59			}
60
61			###################################create tables############################3
62
63	dpavlin	14	if ($opt{c}) {
64	dpavlin	49	sub do_index {
65			my $index = shift \|\| return;
66			my ($table,$col,$unique) = split(/_/, $index);
67			$unique \|\|= '';
68	dpavlin	52	$index =~ s/,/_/g;
69	dpavlin	49	$dbh->do(qq{ create $unique index $index on $table($col) });
70			}
71
72	dpavlin	14	print "creating tables...\n";
73	dpavlin	6
74	dpavlin	14	$dbh->do(qq{
75			create table hosts (
76	dpavlin	49	ID SERIAL PRIMARY KEY,
77	dpavlin	14	name VARCHAR(30) NOT NULL,
78			IP VARCHAR(15)
79	dpavlin	6	);
80	dpavlin	14	});
81	dpavlin	6
82	dpavlin	14	$dbh->do(qq{
83			create table shares (
84	dpavlin	49	ID SERIAL PRIMARY KEY,
85	dpavlin	14	hostID INTEGER NOT NULL references hosts(id),
86			name VARCHAR(30) NOT NULL,
87			share VARCHAR(200) NOT NULL,
88			localpath VARCHAR(200)
89	dpavlin	6	);
90	dpavlin	14	});
91	dpavlin	6
92	dpavlin	14	$dbh->do(qq{
93			create table backups (
94			hostID INTEGER NOT NULL references hosts(id),
95			num INTEGER NOT NULL,
96	dpavlin	49	date integer NOT NULL,
97			type CHAR(4) not null,
98	dpavlin	65	shareID integer not null references shares(id),
99	dpavlin	66	size integer not null,
100	dpavlin	65	PRIMARY KEY(hostID, num, shareID)
101	dpavlin	6	);
102	dpavlin	14	});
103	dpavlin	6
104	dpavlin	65	#do_index('backups_hostid,num_unique');
105	dpavlin	49
106	dpavlin	14	$dbh->do(qq{
107			create table dvds (
108	dpavlin	49	ID SERIAL PRIMARY KEY,
109	dpavlin	14	num INTEGER NOT NULL,
110			name VARCHAR(255) NOT NULL,
111			mjesto VARCHAR(255)
112	dpavlin	6	);
113	dpavlin	14	});
114	dpavlin	6
115	dpavlin	14	$dbh->do(qq{
116			create table files (
117	dpavlin	49	ID SERIAL PRIMARY KEY,
118	dpavlin	14	shareID INTEGER NOT NULL references shares(id),
119	dpavlin	52	backupNum INTEGER NOT NULL,
120	dpavlin	14	name VARCHAR(255) NOT NULL,
121			path VARCHAR(255) NOT NULL,
122	dpavlin	49	date integer NOT NULL,
123	dpavlin	14	type INTEGER NOT NULL,
124			size INTEGER NOT NULL,
125			dvdid INTEGER references dvds(id)
126	dpavlin	6	);
127	dpavlin	14	});
128	dpavlin	6
129	dpavlin	49	print "creating indexes:";
130	dpavlin	6
131	dpavlin	14	foreach my $index (qw(
132			hosts_name
133			backups_hostID
134			backups_num
135			shares_hostID
136			shares_name
137			files_shareID
138			files_path
139			files_name
140			files_date
141			files_size
142			)) {
143	dpavlin	49	print " $index";
144			do_index($index);
145	dpavlin	14	}
146	dpavlin	49	print "...\n";
147	dpavlin	14
148	dpavlin	49	$dbh->commit;
149	dpavlin	14
150			}
151
152			if ($opt{d}) {
153			print "deleting ";
154	dpavlin	49	foreach my $table (qw(files dvds backups shares hosts)) {
155	dpavlin	14	print "$table ";
156			$dbh->do(qq{ DELETE FROM $table });
157			}
158			print " done...\n";
159	dpavlin	49
160	dpavlin	51	$dbh->commit;
161	dpavlin	14	}
162
163	dpavlin	32	if ($opt{v}) {
164			print "Debug level at $opt{v}\n";
165			$debug = $opt{v};
166			}
167
168	dpavlin	6	#################################INSERT VALUES#############################
169
170			# get hosts
171	dpavlin	8	$hosts = $bpc->HostInfoRead();
172	dpavlin	6	my $hostID;
173			my $shareID;
174
175	dpavlin	14	my $sth;
176	dpavlin	6
177	dpavlin	14	$sth->{insert_hosts} = $dbh->prepare(qq{
178			INSERT INTO hosts (name, IP) VALUES (?,?)
179			});
180	dpavlin	6
181	dpavlin	14	$sth->{hosts_by_name} = $dbh->prepare(qq{
182			SELECT ID FROM hosts WHERE name=?
183			});
184
185	dpavlin	65	$sth->{backups_count} = $dbh->prepare(qq{
186	dpavlin	14	SELECT COUNT(*)
187			FROM backups
188	dpavlin	65	WHERE hostID=? AND num=? AND shareid=?
189	dpavlin	14	});
190
191			$sth->{insert_backups} = $dbh->prepare(qq{
192	dpavlin	66	INSERT INTO backups (hostID, num, date, type, shareid, size)
193			VALUES (?,?,?,?,?,?)
194	dpavlin	14	});
195
196			$sth->{insert_files} = $dbh->prepare(qq{
197			INSERT INTO files
198	dpavlin	62	(shareID, backupNum, name, path, date, type, size)
199			VALUES (?,?,?,?,?,?,?)
200	dpavlin	14	});
201
202	dpavlin	50	sub fmt_time {
203			my $t = shift \|\| return;
204			my $out = "";
205			my ($ss,$mm,$hh) = gmtime($t);
206			$out .= "${hh}h" if ($hh);
207			$out .= sprintf("%02d:%02d", $mm,$ss);
208			return $out;
209			}
210
211	dpavlin	14	foreach my $host_key (keys %{$hosts}) {
212
213			my $hostname = $hosts->{$host_key}->{'host'} \|\| die "can't find host for $host_key";
214
215			$sth->{hosts_by_name}->execute($hosts->{$host_key}->{'host'});
216
217			unless (($hostID) = $sth->{hosts_by_name}->fetchrow_array()) {
218			$sth->{insert_hosts}->execute(
219			$hosts->{$host_key}->{'host'},
220			$hosts->{$host_key}->{'ip'}
221			);
222
223	dpavlin	49	$hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
224	dpavlin	14	}
225
226			print("host ".$hosts->{$host_key}->{'host'}.": ");
227	dpavlin	6
228	dpavlin	14	# get backups for a host
229			my @backups = $bpc->BackupInfoRead($hostname);
230			print scalar @backups, " increments\n";
231	dpavlin	6
232	dpavlin	14	my $inc_nr = 0;
233
234			foreach my $backup (@backups) {
235	dpavlin	40
236	dpavlin	14	$inc_nr++;
237			last if ($opt{m} && $inc_nr > $opt{m});
238
239			my $backupNum = $backup->{'num'};
240			my @backupShares = ();
241
242	dpavlin	40	print $hosts->{$host_key}->{'host'},
243	dpavlin	44	"\t#$backupNum\t", $backup->{type} \|\| '?', " ",
244	dpavlin	40	$backup->{nFilesNew} \|\| '?', "/", $backup->{nFiles} \|\| '?',
245	dpavlin	57	" files (date: ",
246			strftime($t_fmt,localtime($backup->{startTime})),
247			" dur: ",
248			fmt_time($backup->{endTime} - $backup->{startTime}),
249			")\n";
250	dpavlin	14
251	dpavlin	34	my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
252	dpavlin	14	foreach my $share ($files->shareList($backupNum)) {
253
254	dpavlin	37	my $t = time();
255
256	dpavlin	14	$shareID = getShareID($share, $hostID, $hostname);
257
258	dpavlin	65	$sth->{backups_count}->execute($hostID, $backupNum, $shareID);
259			my ($count) = $sth->{backups_count}->fetchrow_array();
260			# skip if allready in database!
261			next if ($count > 0);
262
263			# dump some log
264			print strftime($t_fmt,localtime())," ", $share;
265
266	dpavlin	66	my ($f, $nf, $d, $nd, $size) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
267	dpavlin	65
268			$sth->{insert_backups}->execute(
269			$hostID,
270			$backupNum,
271			$backup->{'endTime'},
272			$backup->{'type'},
273	dpavlin	66	$shareID,
274			$size,
275	dpavlin	65	);
276
277			print " commit";
278			$dbh->commit();
279
280	dpavlin	50	my $dur = (time() - $t) \|\| 1;
281	dpavlin	66	printf(" %d/%d files %d/%d dirs %0.2f MB [%.2f/s dur: %s]\n",
282	dpavlin	37	$nf, $f, $nd, $d,
283	dpavlin	66	($size / 1024 / 1024),
284	dpavlin	50	( ($f+$d) / $dur ),
285			fmt_time($dur)
286	dpavlin	37	);
287	dpavlin	14	}
288	dpavlin	29
289	dpavlin	6	}
290			}
291	dpavlin	14	undef $sth;
292	dpavlin	6	$dbh->commit();
293			$dbh->disconnect();
294
295	dpavlin	51	print "total duration: ",fmt_time(time() - $start_t),"\n";
296
297	dpavlin	38	$pidfile->remove;
298
299	dpavlin	14	sub getShareID() {
300	dpavlin	6
301	dpavlin	14	my ($share, $hostID, $hostname) = @_;
302
303			$sth->{share_id} \|\|= $dbh->prepare(qq{
304			SELECT ID FROM shares WHERE hostID=? AND name=?
305			});
306
307			$sth->{share_id}->execute($hostID,$share);
308
309			my ($id) = $sth->{share_id}->fetchrow_array();
310
311			return $id if (defined($id));
312
313			$sth->{insert_share} \|\|= $dbh->prepare(qq{
314			INSERT INTO shares
315			(hostID,name,share,localpath)
316			VALUES (?,?,?,?)
317			});
318
319	dpavlin	25	my $drop_down = $hostname . '/' . $share;
320			$drop_down =~ s#//+#/#g;
321
322			$sth->{insert_share}->execute($hostID,$share, $drop_down ,undef);
323	dpavlin	49	return $dbh->last_insert_id(undef,undef,'shares',undef);
324	dpavlin	6	}
325
326	dpavlin	14	sub found_in_db {
327
328	dpavlin	48	my @data = @_;
329			shift @data;
330	dpavlin	14
331	dpavlin	48	my ($key, $shareID,undef,$name,$path,undef,$date,undef,$size) = @_;
332
333			return $beenThere->{$key} if (defined($beenThere->{$key}));
334
335	dpavlin	14	$sth->{file_in_db} \|\|= $dbh->prepare(qq{
336	dpavlin	48	SELECT 1 FROM files
337	dpavlin	14	WHERE shareID = ? and
338			path = ? and
339			name = ? and
340			date = ? and
341			size = ?
342			});
343
344			my @param = ($shareID,$path,$name,$date,$size);
345			$sth->{file_in_db}->execute(@param);
346	dpavlin	48	my $rows = $sth->{file_in_db}->rows;
347	dpavlin	45	print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
348	dpavlin	48
349			$beenThere->{$key}++;
350
351			$sth->{'insert_files'}->execute(@data) unless ($rows);
352	dpavlin	14	return $rows;
353	dpavlin	6	}
354
355			####################################################
356			# recursing through filesystem structure and #
357			# and returning flattened files list #
358			####################################################
359	dpavlin	14	sub recurseDir($$$$$$$$) {
360	dpavlin	6
361	dpavlin	35	my ($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID) = @_;
362	dpavlin	14
363	dpavlin	44	print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);
364	dpavlin	29
365	dpavlin	66	my ($nr_files, $new_files, $nr_dirs, $new_dirs, $size) = (0,0,0,0,0);
366	dpavlin	14
367	dpavlin	27	{ # scope
368	dpavlin	29	my @stack;
369	dpavlin	14
370	dpavlin	45	print STDERR "# dirAttrib($backupNum, $share, $dir)\n" if ($debug >= 2);
371	dpavlin	27	my $filesInBackup = $files->dirAttrib($backupNum, $share, $dir);
372	dpavlin	14
373	dpavlin	27	# first, add all the entries in current directory
374			foreach my $path_key (keys %{$filesInBackup}) {
375	dpavlin	66	print STDERR "# file ",Dumper($filesInBackup->{$path_key}),"\n" if ($debug >= 3);
376	dpavlin	27	my @data = (
377			$shareID,
378			$backupNum,
379			$path_key,
380			$filesInBackup->{$path_key}->{'relPath'},
381			$filesInBackup->{$path_key}->{'mtime'},
382			$filesInBackup->{$path_key}->{'type'},
383			$filesInBackup->{$path_key}->{'size'}
384			);
385
386			my $key = join(" ", (
387			$shareID,
388			$dir,
389			$path_key,
390			$filesInBackup->{$path_key}->{'mtime'},
391			$filesInBackup->{$path_key}->{'size'}
392			));
393
394
395	dpavlin	48	if (! defined($beenThere->{$key}) && ! found_in_db($key, @data)) {
396	dpavlin	30	print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
397	dpavlin	48
398	dpavlin	27	if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
399			$new_dirs++;
400	dpavlin	30	print STDERR " dir\n" if ($debug >= 2);
401	dpavlin	27	} else {
402			$new_files++;
403	dpavlin	30	print STDERR " file\n" if ($debug >= 2);
404	dpavlin	27	}
405	dpavlin	66	$size += $filesInBackup->{$path_key}->{'size'} \|\| 0;
406	dpavlin	27	}
407
408	dpavlin	14	if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
409	dpavlin	27	$nr_dirs++;
410
411	dpavlin	29	my $full_path = $dir . '/' . $path_key;
412			push @stack, $full_path;
413	dpavlin	30	print STDERR "### store to stack: $full_path\n" if ($debug >= 3);
414	dpavlin	29
415	dpavlin	27	# my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $backups, $backupNum, $share, $path_key, $shareID) unless ($beenThere->{$key});
416			#
417			# $nr_files += $f;
418			# $new_files += $nf;
419			# $nr_dirs += $d;
420			# $new_dirs += $nd;
421
422	dpavlin	14	} else {
423	dpavlin	27	$nr_files++;
424	dpavlin	14	}
425			}
426
427	dpavlin	30	print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);
428	dpavlin	14
429	dpavlin	29	while ( my $dir = shift @stack ) {
430	dpavlin	66	my ($f,$nf,$d,$nd, $s) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
431	dpavlin	30	print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
432	dpavlin	29	$nr_files += $f;
433			$new_files += $nf;
434			$nr_dirs += $d;
435			$new_dirs += $nd;
436	dpavlin	66	$size += $s;
437	dpavlin	29	}
438	dpavlin	14	}
439
440	dpavlin	66	return ($nr_files, $new_files, $nr_dirs, $new_dirs, $size);
441	dpavlin	6	}
442