/[couchdb]/scripts/reblog2couchdb.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /scripts/reblog2couchdb.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 2 by dpavlin, Tue Aug 5 13:28:54 2008 UTC revision 7 by dpavlin, Tue Aug 5 16:36:57 2008 UTC
# Line 9  $|++; Line 9  $|++;
9    
10  my $database = 'reblog';  my $database = 'reblog';
11    
12  my $connect = "DBI:mysql:database=$database";  my $dbi = "DBI:mysql:database=$database";
13  #$connect = "DBI:mysql:database=$database;host=localhost;port=13306";  $dbi .= ";host=127.0.0.1;port=13306";   # XXX over ssh
14    
 my $dbh = DBI->connect($connect,"","") || die $DBI::errstr;  
   
 # select all posts which have been read or unread  
15  my $sql = qq{  my $sql = qq{
16          select          select
17                  *                  items.id as _id,
18                    items.*,
19                    feeds.url as feed_url,
20                    feeds.title as feed_title,
21                    feeds.link as feed_link,
22                    feeds.description as feed_description
23          from items          from items
24  --      join items_userdata on id=item_id          join items_userdata on items.id = item_id
25  --      where label = 'read' and value_numeric = ?          join feeds on items.feed_id = feeds.id
26  --      limit 3          where items.id > ?
27          order by id asc          order by items.id asc
28            limit 5000
29  };  };
30    
 my $sth = $dbh->prepare($sql) || die $dbh->errstr();  
 $sth->execute( 1 ) || die $sth->errstr();  
   
 print "found ",$sth->rows," items to process...";  
31    
32  my $c = CouchDB::Client->new(uri => 'http://localhost:5984/');  my $c = CouchDB::Client->new(uri => 'http://localhost:5984/');
33    
34  $c->testConnection or die "The server cannot be reached";  $c->testConnection or die "The server cannot be reached";
35  print "Running version " . $c->serverInfo->{version} . "\n";  print "CouchDB version " . $c->serverInfo->{version} . "\n";
36  my $db = $c->newDB( $database );  my $db = $c->newDB( $database );
37  $db->create unless $c->dbExists( $database );  $db->create unless $c->dbExists( $database );
38    
39  my @docs = $db->listDocs;  my $status = $db->newDoc( '_sync' );
40  my $row_id = shift @docs || 0;  eval { $status->retrieve };
41    $status->create if $@;
42    
43    print "status ",dump( $status->{data} ), "\n";
44    
45    my $last_row = $status->{data}->{last_row_id} || 0;
46    
47    sub commit_last_row {
48            warn "commit_last_row $last_row\n";
49            $status->{data}->{last_row_id} = $last_row;
50            $status->update;
51    }
52    
53    my $dbh = DBI->connect($dbi,"","") || die $DBI::errstr;
54    
55    print "Fetching items from $dbi id > $last_row\n";
56    
57    my $sth = $dbh->prepare($sql) || die $dbh->errstr();
58    $sth->execute( $last_row ) || die $sth->errstr();
59    
60    warn dump( $sth->{NAME} );
61    
62    print "found ",$sth->rows," items to process...";
63    
64  my $pk = 'id';  my $pk = 'id';
65    
66    my $count = 0;
67    
68  while (my $row = $sth->fetchrow_hashref() ) {  while (my $row = $sth->fetchrow_hashref() ) {
69          my $_id = $row->{$pk};          my $_id = $row->{_id} || die "row needs _id";
70          my $doc = $db->newDoc( $_id );          my $doc = $db->newDoc( $_id );
71          $doc->{data} = $row;  
72            sub row2doc {
73                    my ( $row, $doc ) = @_;
74                    my $a = delete( $row->{xml} );
75                    $doc->addAttachment( 'item.xml', 'application/xhtml+xml', $a ) if $a;
76                    my $a = delete( $row->{content} );
77                    $doc->addAttachment( 'content.html', 'text/html', $a ) if $a;
78                    $doc->{data} = $row;
79                    return $doc;
80            }
81    
82            row2doc( $row, $doc );
83    
84          eval { $doc->create };          eval { $doc->create };
85          if ( $@ ) {          if ( $@ ) {
86                  warn "can't create $_id $@";                  $doc->retrieve;
87                  eval { $doc->update };                  row2doc( $row, $doc )->update;
88                  warn $@ ? "can't update $_id $@" : "updated $_id";  #               eval { $doc->update };
89                    warn $@ ? "ERROR $_id $@\n" : "updated $_id\n";
90          } else {          } else {
91                  warn "created ",dump( $row );                  warn "created $_id\n";
92          }          }
93    
94            $last_row = $row->{id};
95            $count++;
96    
97            commit_last_row if $count % 100 == 0    # checkpoint every 100 records
98  }  }
99    
100    commit_last_row;
101    
102  __END__  __END__
103    
104  $sql = qq{  $sql = qq{
Legend:
Removed from v.2  
changed lines
  Added in v.7
  ViewVC Help
Powered by ViewVC 1.1.26