Annotation of /trunk/bin/BackupPC_updatedb

#!/usr/local/bin/perl -w

use strict;
use lib "__INSTALLDIR__/lib";

use DBI;
use BackupPC::Lib;
use BackupPC::View;
use Data::Dumper;
use Getopt::Std;
use Time::HiRes qw/time/;
use File::Pid;
use POSIX qw/strftime/;

use constant BPC_FTYPE_DIR => 5;

my $debug = 0;
$|=1;

my $start_t = time();

my $pidfile = new File::Pid;

if (my $pid = $pidfile->running ) {
        die "$0 already running: $pid\n";
} elsif ($pidfile->pid ne $$) {
        $pidfile->remove;
        $pidfile = new File::Pid;
}
$pidfile->write;
print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";

my $t_fmt = '%Y-%m-%d %H:%M:%S';

my $hosts;
my $bpc = BackupPC::Lib->new || die;
my %Conf = $bpc->Conf();
my $TopDir = $bpc->TopDir();
my $beenThere = {};

my $dsn = $Conf{SearchDSN} || die "Need SearchDSN in config.pl\n";
my $user = $Conf{SearchUser} || '';

my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });

my %opt;

if ( !getopts("cdm:v:", \%opt ) ) {
        print STDERR <<EOF;
usage: $0 [-c|-d] [-m num] [-v|-v level]

Options:
        -c      create database on first use
        -d      delete database before import
        -m num  import just num increments for one host
        -v num  set verbosity (debug) level (default $debug)
EOF
        exit 1;
}

###################################create tables############################3

if ($opt{c}) {
        sub do_index {
                my $index = shift || return;
                my ($table,$col,$unique) = split(/_/, $index);
                $unique ||= '';
                $index =~ s/,/_/g;
                $dbh->do(qq{ create $unique index $index on $table($col) });
        }

        print "creating tables...\n";
      
        $dbh->do(qq{
                create table hosts (
                        ID      SERIAL          PRIMARY KEY,
                        name    VARCHAR(30)     NOT NULL,
                        IP      VARCHAR(15)
                );            
        });
              
        $dbh->do(qq{
                create table shares (
                        ID      SERIAL          PRIMARY KEY,
                        hostID  INTEGER         NOT NULL references hosts(id),
                        name    VARCHAR(30)     NOT NULL,
                        share   VARCHAR(200)    NOT NULL,
                        localpath VARCHAR(200)      
                );            
        });
        
        $dbh->do(qq{
                create table backups (
                        hostID  INTEGER         NOT NULL references hosts(id),
                        num     INTEGER         NOT NULL,
                        date    integer         NOT NULL, 
                        type    CHAR(4)         not null,
                        shareID integer         not null references shares(id),
                        PRIMARY KEY(hostID, num, shareID) 
                );            
        });

        #do_index('backups_hostid,num_unique');

        $dbh->do(qq{
                create table dvds (
                        ID      SERIAL          PRIMARY KEY, 
                        num     INTEGER         NOT NULL,
                        name    VARCHAR(255)    NOT NULL,
                        mjesto  VARCHAR(255)
                );
        });

        $dbh->do(qq{     
                create table files (
                        ID      SERIAL          PRIMARY KEY,  
                        shareID INTEGER         NOT NULL references shares(id),
                        backupNum  INTEGER      NOT NULL,
                        name       VARCHAR(255) NOT NULL,
                        path       VARCHAR(255) NOT NULL,
                        date       integer      NOT NULL,
                        type       INTEGER      NOT NULL,
                        size       INTEGER      NOT NULL,
                        dvdid      INTEGER      references dvds(id)     
                );
        });

        print "creating indexes:";

        foreach my $index (qw(
                hosts_name
                backups_hostID
                backups_num
                shares_hostID
                shares_name
                files_shareID
                files_path
                files_name
                files_date
                files_size
        )) {
                print " $index";
                do_index($index);
        }
        print "...\n";

        $dbh->commit;

}

if ($opt{d}) {
        print "deleting ";
        foreach my $table (qw(files dvds backups shares hosts)) {
                print "$table ";
                $dbh->do(qq{ DELETE FROM $table });
        }
        print " done...\n";

        $dbh->commit;
}

if ($opt{v}) {
        print "Debug level at $opt{v}\n";
        $debug = $opt{v};
}

#################################INSERT VALUES#############################

# get hosts
$hosts = $bpc->HostInfoRead();
my $hostID;
my $shareID;

my $sth;

$sth->{insert_hosts} = $dbh->prepare(qq{
INSERT INTO hosts (name, IP) VALUES (?,?)
});

$sth->{hosts_by_name} = $dbh->prepare(qq{
SELECT ID FROM hosts WHERE name=?
});

$sth->{backups_count} = $dbh->prepare(qq{
SELECT COUNT(*)
FROM backups
WHERE hostID=? AND num=? AND shareid=?
});

$sth->{insert_backups} = $dbh->prepare(qq{
INSERT INTO backups (hostID, num, date, type, shareid)
VALUES (?,?,?,?,?)
});

$sth->{insert_files} = $dbh->prepare(qq{
INSERT INTO files
        (shareID, backupNum, name, path, date, type, size)
        VALUES (?,?,?,?,?,?,?)
});

sub fmt_time {
        my $t = shift || return;
        my $out = "";
        my ($ss,$mm,$hh) = gmtime($t);
        $out .= "${hh}h" if ($hh);
        $out .= sprintf("%02d:%02d", $mm,$ss);
        return $out;
}

foreach my $host_key (keys %{$hosts}) {

        my $hostname = $hosts->{$host_key}->{'host'} || die "can't find host for $host_key";

        $sth->{hosts_by_name}->execute($hosts->{$host_key}->{'host'});

        unless (($hostID) = $sth->{hosts_by_name}->fetchrow_array()) {
                $sth->{insert_hosts}->execute(
                        $hosts->{$host_key}->{'host'},
                        $hosts->{$host_key}->{'ip'}
                );

                $hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
        }

        print("host ".$hosts->{$host_key}->{'host'}.": ");
 
        # get backups for a host
        my @backups = $bpc->BackupInfoRead($hostname);
        print scalar @backups, " increments\n";

        my $inc_nr = 0;

        foreach my $backup (@backups) {

                $inc_nr++;
                last if ($opt{m} && $inc_nr > $opt{m});

                my $backupNum = $backup->{'num'};
                my @backupShares = ();

                print $hosts->{$host_key}->{'host'},
                        "\t#$backupNum\t", $backup->{type} || '?', " ",
                        $backup->{nFilesNew} || '?', "/", $backup->{nFiles} || '?',
                        " files (date: ",
                        strftime($t_fmt,localtime($backup->{startTime})),
                        " dur: ",
                        fmt_time($backup->{endTime} - $backup->{startTime}),
                        ")\n";

                my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
                foreach my $share ($files->shareList($backupNum)) {

                        my $t = time();

                        $shareID = getShareID($share, $hostID, $hostname);
                
                        $sth->{backups_count}->execute($hostID, $backupNum, $shareID);
                        my ($count) = $sth->{backups_count}->fetchrow_array();
                        # skip if allready in database!
                        next if ($count > 0);

                        # dump some log
                        print strftime($t_fmt,localtime())," ", $share;

                        my ($f, $nf, $d, $nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);

                        $sth->{insert_backups}->execute(
                                $hostID,
                                $backupNum,
                                $backup->{'endTime'},
                                $backup->{'type'},
                                $shareID
                        );

                        print " commit";
                        $dbh->commit();

                        my $dur = (time() - $t) || 1;
                        printf(" %d/%d files %d/%d dirs [%.2f/s dur: %s]\n",
                                $nf, $f, $nd, $d,
                                ( ($f+$d) / $dur ),
                                fmt_time($dur)
                        );
                }

        }
}
undef $sth;
$dbh->commit();
$dbh->disconnect();

print "total duration: ",fmt_time(time() - $start_t),"\n";

$pidfile->remove;

sub getShareID() {

        my ($share, $hostID, $hostname) = @_;

        $sth->{share_id} ||= $dbh->prepare(qq{
                SELECT ID FROM shares WHERE hostID=? AND name=?
        });

        $sth->{share_id}->execute($hostID,$share);

        my ($id) = $sth->{share_id}->fetchrow_array();

        return $id if (defined($id));

        $sth->{insert_share} ||= $dbh->prepare(qq{
                INSERT INTO shares 
                        (hostID,name,share,localpath) 
                VALUES (?,?,?,?)
        });

        my $drop_down = $hostname . '/' . $share;
        $drop_down =~ s#//+#/#g;

        $sth->{insert_share}->execute($hostID,$share, $drop_down ,undef);
        return $dbh->last_insert_id(undef,undef,'shares',undef);
}

sub found_in_db {

        my @data = @_;
        shift @data;

        my ($key, $shareID,undef,$name,$path,undef,$date,undef,$size) = @_;

        return $beenThere->{$key} if (defined($beenThere->{$key}));

        $sth->{file_in_db} ||= $dbh->prepare(qq{
                SELECT 1 FROM files
                WHERE shareID = ? and
                        path = ? and 
                        name = ? and
                        date = ? and
                        size = ?
        });

        my @param = ($shareID,$path,$name,$date,$size);
        $sth->{file_in_db}->execute(@param);
        my $rows = $sth->{file_in_db}->rows;
        print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);

        $beenThere->{$key}++;

        $sth->{'insert_files'}->execute(@data) unless ($rows);
        return $rows;
}

####################################################
# recursing through filesystem structure and       #
# and returning flattened files list               #
####################################################
sub recurseDir($$$$$$$$) {

        my ($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID) = @_;

        print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);

        my ($nr_files, $new_files, $nr_dirs, $new_dirs) = (0,0,0,0);

        { # scope
                my @stack;

                print STDERR "# dirAttrib($backupNum, $share, $dir)\n" if ($debug >= 2);
                my $filesInBackup = $files->dirAttrib($backupNum, $share, $dir);

                # first, add all the entries in current directory
                foreach my $path_key (keys %{$filesInBackup}) {
                        my @data = (
                                $shareID,
                                $backupNum,
                                $path_key,
                                $filesInBackup->{$path_key}->{'relPath'},
                                $filesInBackup->{$path_key}->{'mtime'},
                                $filesInBackup->{$path_key}->{'type'},
                                $filesInBackup->{$path_key}->{'size'}
                        );

                        my $key = join(" ", (
                                $shareID,
                                $dir,
                                $path_key,
                                $filesInBackup->{$path_key}->{'mtime'},
                                $filesInBackup->{$path_key}->{'size'}
                        ));


                        if (! defined($beenThere->{$key}) && ! found_in_db($key, @data)) {
                                print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);

                                if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
                                        $new_dirs++;
                                        print STDERR " dir\n" if ($debug >= 2);
                                } else {
                                        $new_files++;
                                        print STDERR " file\n" if ($debug >= 2);
                                }
                        }

                        if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
                                $nr_dirs++;

                                my $full_path = $dir . '/' . $path_key;
                                push @stack, $full_path;
                                print STDERR "### store to stack: $full_path\n" if ($debug >= 3);

#                               my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $backups, $backupNum, $share, $path_key, $shareID) unless ($beenThere->{$key});
#
#                               $nr_files += $f;
#                               $new_files += $nf;
#                               $nr_dirs += $d;
#                               $new_dirs += $nd;

                        } else {
                                $nr_files++;
                        }
                }

                print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);

                while ( my $dir = shift @stack ) {
                        my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
                        print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
                        $nr_files += $f;
                        $new_files += $nf;
                        $nr_dirs += $d;
                        $new_dirs += $nd;
                }
        }

        return ($nr_files, $new_files, $nr_dirs, $new_dirs);
}

1	dpavlin	14	#!/usr/local/bin/perl -w
2	dpavlin	6
3			use strict;
4	dpavlin	48	use lib "__INSTALLDIR__/lib";
5
6	dpavlin	6	use DBI;
7			use BackupPC::Lib;
8			use BackupPC::View;
9			use Data::Dumper;
10			use Getopt::Std;
11	dpavlin	37	use Time::HiRes qw/time/;
12	dpavlin	38	use File::Pid;
13	dpavlin	37	use POSIX qw/strftime/;
14	dpavlin	48
15	dpavlin	6	use constant BPC_FTYPE_DIR => 5;
16
17	dpavlin	30	my $debug = 0;
18	dpavlin	14	$\|=1;
19	dpavlin	6
20	dpavlin	51	my $start_t = time();
21
22	dpavlin	38	my $pidfile = new File::Pid;
23
24			if (my $pid = $pidfile->running ) {
25			die "$0 already running: $pid\n";
26			} elsif ($pidfile->pid ne $$) {
27			$pidfile->remove;
28			$pidfile = new File::Pid;
29			}
30	dpavlin	39	$pidfile->write;
31			print STDERR "$0 using pid ",$pidfile->pid," file ",$pidfile->file,"\n";
32	dpavlin	38
33	dpavlin	37	my $t_fmt = '%Y-%m-%d %H:%M:%S';
34
35	dpavlin	6	my $hosts;
36			my $bpc = BackupPC::Lib->new \|\| die;
37			my %Conf = $bpc->Conf();
38			my $TopDir = $bpc->TopDir();
39	dpavlin	14	my $beenThere = {};
40	dpavlin	6
41	dpavlin	51	my $dsn = $Conf{SearchDSN} \|\| die "Need SearchDSN in config.pl\n";
42			my $user = $Conf{SearchUser} \|\| '';
43	dpavlin	6
44	dpavlin	49	my $dbh = DBI->connect($dsn, $user, "", { RaiseError => 1, AutoCommit => 0 });
45
46	dpavlin	6	my %opt;
47
48	dpavlin	44	if ( !getopts("cdm:v:", \%opt ) ) {
49	dpavlin	6	print STDERR <<EOF;
50	dpavlin	44	usage: $0 [-c\|-d] [-m num] [-v\|-v level]
51	dpavlin	6
52			Options:
53	dpavlin	14	-c create database on first use
54			-d delete database before import
55			-m num import just num increments for one host
56	dpavlin	44	-v num set verbosity (debug) level (default $debug)
57	dpavlin	6	EOF
58			exit 1;
59			}
60
61			###################################create tables############################3
62
63	dpavlin	14	if ($opt{c}) {
64	dpavlin	49	sub do_index {
65			my $index = shift \|\| return;
66			my ($table,$col,$unique) = split(/_/, $index);
67			$unique \|\|= '';
68	dpavlin	52	$index =~ s/,/_/g;
69	dpavlin	49	$dbh->do(qq{ create $unique index $index on $table($col) });
70			}
71
72	dpavlin	14	print "creating tables...\n";
73	dpavlin	6
74	dpavlin	14	$dbh->do(qq{
75			create table hosts (
76	dpavlin	49	ID SERIAL PRIMARY KEY,
77	dpavlin	14	name VARCHAR(30) NOT NULL,
78			IP VARCHAR(15)
79	dpavlin	6	);
80	dpavlin	14	});
81	dpavlin	6
82	dpavlin	14	$dbh->do(qq{
83			create table shares (
84	dpavlin	49	ID SERIAL PRIMARY KEY,
85	dpavlin	14	hostID INTEGER NOT NULL references hosts(id),
86			name VARCHAR(30) NOT NULL,
87			share VARCHAR(200) NOT NULL,
88			localpath VARCHAR(200)
89	dpavlin	6	);
90	dpavlin	14	});
91	dpavlin	6
92	dpavlin	14	$dbh->do(qq{
93			create table backups (
94			hostID INTEGER NOT NULL references hosts(id),
95			num INTEGER NOT NULL,
96	dpavlin	49	date integer NOT NULL,
97			type CHAR(4) not null,
98	dpavlin	65	shareID integer not null references shares(id),
99			PRIMARY KEY(hostID, num, shareID)
100	dpavlin	6	);
101	dpavlin	14	});
102	dpavlin	6
103	dpavlin	65	#do_index('backups_hostid,num_unique');
104	dpavlin	49
105	dpavlin	14	$dbh->do(qq{
106			create table dvds (
107	dpavlin	49	ID SERIAL PRIMARY KEY,
108	dpavlin	14	num INTEGER NOT NULL,
109			name VARCHAR(255) NOT NULL,
110			mjesto VARCHAR(255)
111	dpavlin	6	);
112	dpavlin	14	});
113	dpavlin	6
114	dpavlin	14	$dbh->do(qq{
115			create table files (
116	dpavlin	49	ID SERIAL PRIMARY KEY,
117	dpavlin	14	shareID INTEGER NOT NULL references shares(id),
118	dpavlin	52	backupNum INTEGER NOT NULL,
119	dpavlin	14	name VARCHAR(255) NOT NULL,
120			path VARCHAR(255) NOT NULL,
121	dpavlin	49	date integer NOT NULL,
122	dpavlin	14	type INTEGER NOT NULL,
123			size INTEGER NOT NULL,
124			dvdid INTEGER references dvds(id)
125	dpavlin	6	);
126	dpavlin	14	});
127	dpavlin	6
128	dpavlin	49	print "creating indexes:";
129	dpavlin	6
130	dpavlin	14	foreach my $index (qw(
131			hosts_name
132			backups_hostID
133			backups_num
134			shares_hostID
135			shares_name
136			files_shareID
137			files_path
138			files_name
139			files_date
140			files_size
141			)) {
142	dpavlin	49	print " $index";
143			do_index($index);
144	dpavlin	14	}
145	dpavlin	49	print "...\n";
146	dpavlin	14
147	dpavlin	49	$dbh->commit;
148	dpavlin	14
149			}
150
151			if ($opt{d}) {
152			print "deleting ";
153	dpavlin	49	foreach my $table (qw(files dvds backups shares hosts)) {
154	dpavlin	14	print "$table ";
155			$dbh->do(qq{ DELETE FROM $table });
156			}
157			print " done...\n";
158	dpavlin	49
159	dpavlin	51	$dbh->commit;
160	dpavlin	14	}
161
162	dpavlin	32	if ($opt{v}) {
163			print "Debug level at $opt{v}\n";
164			$debug = $opt{v};
165			}
166
167	dpavlin	6	#################################INSERT VALUES#############################
168
169			# get hosts
170	dpavlin	8	$hosts = $bpc->HostInfoRead();
171	dpavlin	6	my $hostID;
172			my $shareID;
173
174	dpavlin	14	my $sth;
175	dpavlin	6
176	dpavlin	14	$sth->{insert_hosts} = $dbh->prepare(qq{
177			INSERT INTO hosts (name, IP) VALUES (?,?)
178			});
179	dpavlin	6
180	dpavlin	14	$sth->{hosts_by_name} = $dbh->prepare(qq{
181			SELECT ID FROM hosts WHERE name=?
182			});
183
184	dpavlin	65	$sth->{backups_count} = $dbh->prepare(qq{
185	dpavlin	14	SELECT COUNT(*)
186			FROM backups
187	dpavlin	65	WHERE hostID=? AND num=? AND shareid=?
188	dpavlin	14	});
189
190			$sth->{insert_backups} = $dbh->prepare(qq{
191	dpavlin	65	INSERT INTO backups (hostID, num, date, type, shareid)
192			VALUES (?,?,?,?,?)
193	dpavlin	14	});
194
195			$sth->{insert_files} = $dbh->prepare(qq{
196			INSERT INTO files
197	dpavlin	62	(shareID, backupNum, name, path, date, type, size)
198			VALUES (?,?,?,?,?,?,?)
199	dpavlin	14	});
200
201	dpavlin	50	sub fmt_time {
202			my $t = shift \|\| return;
203			my $out = "";
204			my ($ss,$mm,$hh) = gmtime($t);
205			$out .= "${hh}h" if ($hh);
206			$out .= sprintf("%02d:%02d", $mm,$ss);
207			return $out;
208			}
209
210	dpavlin	14	foreach my $host_key (keys %{$hosts}) {
211
212			my $hostname = $hosts->{$host_key}->{'host'} \|\| die "can't find host for $host_key";
213
214			$sth->{hosts_by_name}->execute($hosts->{$host_key}->{'host'});
215
216			unless (($hostID) = $sth->{hosts_by_name}->fetchrow_array()) {
217			$sth->{insert_hosts}->execute(
218			$hosts->{$host_key}->{'host'},
219			$hosts->{$host_key}->{'ip'}
220			);
221
222	dpavlin	49	$hostID = $dbh->last_insert_id(undef,undef,'hosts',undef);
223	dpavlin	14	}
224
225			print("host ".$hosts->{$host_key}->{'host'}.": ");
226	dpavlin	6
227	dpavlin	14	# get backups for a host
228			my @backups = $bpc->BackupInfoRead($hostname);
229			print scalar @backups, " increments\n";
230	dpavlin	6
231	dpavlin	14	my $inc_nr = 0;
232
233			foreach my $backup (@backups) {
234	dpavlin	40
235	dpavlin	14	$inc_nr++;
236			last if ($opt{m} && $inc_nr > $opt{m});
237
238			my $backupNum = $backup->{'num'};
239			my @backupShares = ();
240
241	dpavlin	40	print $hosts->{$host_key}->{'host'},
242	dpavlin	44	"\t#$backupNum\t", $backup->{type} \|\| '?', " ",
243	dpavlin	40	$backup->{nFilesNew} \|\| '?', "/", $backup->{nFiles} \|\| '?',
244	dpavlin	57	" files (date: ",
245			strftime($t_fmt,localtime($backup->{startTime})),
246			" dur: ",
247			fmt_time($backup->{endTime} - $backup->{startTime}),
248			")\n";
249	dpavlin	14
250	dpavlin	34	my $files = BackupPC::View->new($bpc, $hostname, \@backups, 1);
251	dpavlin	14	foreach my $share ($files->shareList($backupNum)) {
252
253	dpavlin	37	my $t = time();
254
255	dpavlin	14	$shareID = getShareID($share, $hostID, $hostname);
256
257	dpavlin	65	$sth->{backups_count}->execute($hostID, $backupNum, $shareID);
258			my ($count) = $sth->{backups_count}->fetchrow_array();
259			# skip if allready in database!
260			next if ($count > 0);
261
262			# dump some log
263			print strftime($t_fmt,localtime())," ", $share;
264
265	dpavlin	35	my ($f, $nf, $d, $nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, "", $shareID);
266	dpavlin	65
267			$sth->{insert_backups}->execute(
268			$hostID,
269			$backupNum,
270			$backup->{'endTime'},
271			$backup->{'type'},
272			$shareID
273			);
274
275			print " commit";
276			$dbh->commit();
277
278	dpavlin	50	my $dur = (time() - $t) \|\| 1;
279			printf(" %d/%d files %d/%d dirs [%.2f/s dur: %s]\n",
280	dpavlin	37	$nf, $f, $nd, $d,
281	dpavlin	50	( ($f+$d) / $dur ),
282			fmt_time($dur)
283	dpavlin	37	);
284	dpavlin	14	}
285	dpavlin	29
286	dpavlin	6	}
287			}
288	dpavlin	14	undef $sth;
289	dpavlin	6	$dbh->commit();
290			$dbh->disconnect();
291
292	dpavlin	51	print "total duration: ",fmt_time(time() - $start_t),"\n";
293
294	dpavlin	38	$pidfile->remove;
295
296	dpavlin	14	sub getShareID() {
297	dpavlin	6
298	dpavlin	14	my ($share, $hostID, $hostname) = @_;
299
300			$sth->{share_id} \|\|= $dbh->prepare(qq{
301			SELECT ID FROM shares WHERE hostID=? AND name=?
302			});
303
304			$sth->{share_id}->execute($hostID,$share);
305
306			my ($id) = $sth->{share_id}->fetchrow_array();
307
308			return $id if (defined($id));
309
310			$sth->{insert_share} \|\|= $dbh->prepare(qq{
311			INSERT INTO shares
312			(hostID,name,share,localpath)
313			VALUES (?,?,?,?)
314			});
315
316	dpavlin	25	my $drop_down = $hostname . '/' . $share;
317			$drop_down =~ s#//+#/#g;
318
319			$sth->{insert_share}->execute($hostID,$share, $drop_down ,undef);
320	dpavlin	49	return $dbh->last_insert_id(undef,undef,'shares',undef);
321	dpavlin	6	}
322
323	dpavlin	14	sub found_in_db {
324
325	dpavlin	48	my @data = @_;
326			shift @data;
327	dpavlin	14
328	dpavlin	48	my ($key, $shareID,undef,$name,$path,undef,$date,undef,$size) = @_;
329
330			return $beenThere->{$key} if (defined($beenThere->{$key}));
331
332	dpavlin	14	$sth->{file_in_db} \|\|= $dbh->prepare(qq{
333	dpavlin	48	SELECT 1 FROM files
334	dpavlin	14	WHERE shareID = ? and
335			path = ? and
336			name = ? and
337			date = ? and
338			size = ?
339			});
340
341			my @param = ($shareID,$path,$name,$date,$size);
342			$sth->{file_in_db}->execute(@param);
343	dpavlin	48	my $rows = $sth->{file_in_db}->rows;
344	dpavlin	45	print STDERR "## found_in_db ",( $rows ? '+' : '-' ), join(" ",@param), "\n" if ($debug >= 3);
345	dpavlin	48
346			$beenThere->{$key}++;
347
348			$sth->{'insert_files'}->execute(@data) unless ($rows);
349	dpavlin	14	return $rows;
350	dpavlin	6	}
351
352			####################################################
353			# recursing through filesystem structure and #
354			# and returning flattened files list #
355			####################################################
356	dpavlin	14	sub recurseDir($$$$$$$$) {
357	dpavlin	6
358	dpavlin	35	my ($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID) = @_;
359	dpavlin	14
360	dpavlin	44	print STDERR "\nrecurse($hostname,$backupNum,$share,$dir,$shareID)\n" if ($debug >= 1);
361	dpavlin	29
362	dpavlin	14	my ($nr_files, $new_files, $nr_dirs, $new_dirs) = (0,0,0,0);
363
364	dpavlin	27	{ # scope
365	dpavlin	29	my @stack;
366	dpavlin	14
367	dpavlin	45	print STDERR "# dirAttrib($backupNum, $share, $dir)\n" if ($debug >= 2);
368	dpavlin	27	my $filesInBackup = $files->dirAttrib($backupNum, $share, $dir);
369	dpavlin	14
370	dpavlin	27	# first, add all the entries in current directory
371			foreach my $path_key (keys %{$filesInBackup}) {
372			my @data = (
373			$shareID,
374			$backupNum,
375			$path_key,
376			$filesInBackup->{$path_key}->{'relPath'},
377			$filesInBackup->{$path_key}->{'mtime'},
378			$filesInBackup->{$path_key}->{'type'},
379			$filesInBackup->{$path_key}->{'size'}
380			);
381
382			my $key = join(" ", (
383			$shareID,
384			$dir,
385			$path_key,
386			$filesInBackup->{$path_key}->{'mtime'},
387			$filesInBackup->{$path_key}->{'size'}
388			));
389
390
391	dpavlin	48	if (! defined($beenThere->{$key}) && ! found_in_db($key, @data)) {
392	dpavlin	30	print STDERR "# key: $key [", $beenThere->{$key},"]" if ($debug >= 2);
393	dpavlin	48
394	dpavlin	27	if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
395			$new_dirs++;
396	dpavlin	30	print STDERR " dir\n" if ($debug >= 2);
397	dpavlin	27	} else {
398			$new_files++;
399	dpavlin	30	print STDERR " file\n" if ($debug >= 2);
400	dpavlin	27	}
401			}
402
403	dpavlin	14	if ($filesInBackup->{$path_key}->{'type'} == BPC_FTYPE_DIR) {
404	dpavlin	27	$nr_dirs++;
405
406	dpavlin	29	my $full_path = $dir . '/' . $path_key;
407			push @stack, $full_path;
408	dpavlin	30	print STDERR "### store to stack: $full_path\n" if ($debug >= 3);
409	dpavlin	29
410	dpavlin	27	# my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $backups, $backupNum, $share, $path_key, $shareID) unless ($beenThere->{$key});
411			#
412			# $nr_files += $f;
413			# $new_files += $nf;
414			# $nr_dirs += $d;
415			# $new_dirs += $nd;
416
417	dpavlin	14	} else {
418	dpavlin	27	$nr_files++;
419	dpavlin	14	}
420			}
421
422	dpavlin	30	print STDERR "## STACK ",join(", ", @stack),"\n" if ($debug >= 2);
423	dpavlin	14
424	dpavlin	29	while ( my $dir = shift @stack ) {
425	dpavlin	35	my ($f,$nf,$d,$nd) = recurseDir($bpc, $hostname, $files, $backupNum, $share, $dir, $shareID);
426	dpavlin	30	print STDERR "# $dir f: $f nf: $nf d: $d nd: $nd\n" if ($debug >= 1);
427	dpavlin	29	$nr_files += $f;
428			$new_files += $nf;
429			$nr_dirs += $d;
430			$new_dirs += $nd;
431			}
432	dpavlin	14	}
433
434			return ($nr_files, $new_files, $nr_dirs, $new_dirs);
435	dpavlin	6	}
436