/[couchdb]/scripts/reblog2couchdb.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /scripts/reblog2couchdb.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 2 by dpavlin, Tue Aug 5 13:28:54 2008 UTC revision 10 by dpavlin, Fri Aug 8 19:19:12 2008 UTC
# Line 9  $|++; Line 9  $|++;
9    
10  my $database = 'reblog';  my $database = 'reblog';
11    
12  my $connect = "DBI:mysql:database=$database";  my $dbi = "DBI:mysql:database=$database";
13  #$connect = "DBI:mysql:database=$database;host=localhost;port=13306";  $dbi .= ";host=127.0.0.1;port=13306";   # XXX over ssh
14    
 my $dbh = DBI->connect($connect,"","") || die $DBI::errstr;  
   
 # select all posts which have been read or unread  
15  my $sql = qq{  my $sql = qq{
16          select          select
17                  *                  items.id as _id,
18                    items.*,
19                    feeds.url as feed_url,
20                    feeds.title as feed_title,
21                    feeds.link as feed_link,
22                    feeds.description as feed_description
23    --              t.value_long as tags
24          from items          from items
25  --      join items_userdata on id=item_id          join items_userdata on items.id = item_id
26  --      where label = 'read' and value_numeric = ?          join feeds on items.feed_id = feeds.id
27  --      limit 3  --      left outer join items_userdata as t on items.id = t.item_id and t.label='tags'
28          order by id asc          where items.id > ? and items_userdata.label = 'published' and items_userdata.value_numeric = 1
29            order by items.id asc
30            limit 1000
31  };  };
32    
 my $sth = $dbh->prepare($sql) || die $dbh->errstr();  
 $sth->execute( 1 ) || die $sth->errstr();  
   
 print "found ",$sth->rows," items to process...";  
   
33  my $c = CouchDB::Client->new(uri => 'http://localhost:5984/');  my $c = CouchDB::Client->new(uri => 'http://localhost:5984/');
34    
35  $c->testConnection or die "The server cannot be reached";  $c->testConnection or die "The server cannot be reached";
36  print "Running version " . $c->serverInfo->{version} . "\n";  print "CouchDB version " . $c->serverInfo->{version} . "\n";
37  my $db = $c->newDB( $database );  my $db = $c->newDB( $database );
38  $db->create unless $c->dbExists( $database );  $db->create unless $c->dbExists( $database );
39    
40  my @docs = $db->listDocs;  my $status = $db->newDoc( '_sync' );
41  my $row_id = shift @docs || 0;  eval { $status->retrieve };
42    $status->create if $@;
43    
44    print "status ",dump( $status->{data} ), "\n";
45    
46    my $last_row = $status->{data}->{last_row_id} || 0;
47    
48    sub commit_last_row {
49            warn "commit_last_row $last_row\n";
50            $status->{data}->{last_row_id} = $last_row;
51            $status->update;
52    }
53    
54    my $dbh = DBI->connect($dbi,"","") || die $DBI::errstr;
55    
56    print "Fetching items from $dbi id > $last_row\n";
57    
58    my $sth = $dbh->prepare($sql) || die $dbh->errstr();
59    $sth->execute( $last_row ) || die $sth->errstr();
60    
61    warn dump( $sth->{NAME} );
62    
63    print "found ",$sth->rows," items to process...\n";
64    
65  my $pk = 'id';  my $pk = 'id';
66    
67    my $count = 0;
68    
69  while (my $row = $sth->fetchrow_hashref() ) {  while (my $row = $sth->fetchrow_hashref() ) {
70          my $_id = $row->{$pk};          my $_id = $row->{_id} || die "row needs _id";
71          my $doc = $db->newDoc( $_id );          my $doc = $db->newDoc( $_id );
72          $doc->{data} = $row;  
73            sub row2doc {
74                    my ( $row, $doc ) = @_;
75                    my $a = delete( $row->{xml} );
76                    $doc->addAttachment( 'item.xml', 'application/xhtml+xml', $a ) if $a;
77                    my $a = delete( $row->{content} );
78                    $doc->addAttachment( 'content.html', 'text/html', $a ) if $a;
79                    $doc->{data} = $row;
80                    return $doc;
81            }
82    
83            row2doc( $row, $doc );
84    
85          eval { $doc->create };          eval { $doc->create };
86          if ( $@ ) {          if ( $@ ) {
87                  warn "can't create $_id $@";                  $doc->retrieve;
88                  eval { $doc->update };                  row2doc( $row, $doc )->update;
89                  warn $@ ? "can't update $_id $@" : "updated $_id";  #               eval { $doc->update };
90                    warn $@ ? "$count ERROR $_id $@\n" : "$count updated $_id\n";
91          } else {          } else {
92                  warn "created ",dump( $row );                  warn "$count created $_id\n";
93          }          }
94    
95            $last_row = $row->{id};
96            $count++;
97    
98            commit_last_row if $count % 100 == 0    # checkpoint every 100 records
99  }  }
100    
101    commit_last_row;
102    
103  __END__  __END__
104    
105  $sql = qq{  $sql = qq{
Legend:
Removed from v.2  
changed lines
  Added in v.10
  ViewVC Help
Powered by ViewVC 1.1.26