/[webpac]/trunk2/openisis/openisis.c
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /trunk2/openisis/openisis.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 337 - (hide annotations)
Thu Jun 10 19:22:40 2004 UTC (19 years, 9 months ago) by dpavlin
File MIME type: text/plain
File size: 22423 byte(s)
new trunk for webpac v2

1 dpavlin 237 /*
2     openisis - an open implementation of the CDS/ISIS database
3     Version 0.8.x (patchlevel see file Version)
4     Copyright (C) 2001-2003 by Erik Grziwotz, erik@openisis.org
5    
6     This library is free software; you can redistribute it and/or
7     modify it under the terms of the GNU Lesser General Public
8     License as published by the Free Software Foundation; either
9     version 2.1 of the License, or (at your option) any later version.
10    
11     This library is distributed in the hope that it will be useful,
12     but WITHOUT ANY WARRANTY; without even the implied warranty of
13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14     Lesser General Public License for more details.
15    
16     You should have received a copy of the GNU Lesser General Public
17     License along with this library; if not, write to the Free Software
18     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19    
20     see README for more information
21     EOH */
22    
23     /*
24     $Id: openisis.c,v 1.67 2003/06/03 11:25:02 kripke Exp $
25     main file of openisis executable.
26     */
27    
28     #ifdef HAVE_PTHREAD
29     #define HAVE_THREADS
30     #endif
31    
32     #include <assert.h>
33     #include <stdlib.h> /* free */
34     #include <string.h> /* strcmp */
35     #ifdef WIN32
36     # include <sys/timeb.h>
37     # include <sys/types.h>
38     # define timeval _timeb
39     #else
40     # include <unistd.h> /* gettimeofday */
41     #endif
42     #include <sys/time.h> /* gettimeofday */
43     #ifdef HAVE_PTHREAD
44     #include <pthread.h> /* threaded crashtest */
45     #endif
46    
47     #include "openisis.h"
48    
49     /*
50     temporary library includes for testing of functions
51     that should later be accessible via openisis.h
52     */
53     #include "lio.h"
54     extern Db* ldb_getdb (int dbid);
55    
56    
57     /* ************************************************************
58     private types
59     */
60     typedef enum { /* what to do */
61     DO_DUMP,
62     DO_SCAN, /* simple full scan */
63     DO_SEARCH, /* basic index based searching */
64     DO_TERMS, /* list terms */
65     DO_CHECK, /* check db */
66     DO_PERF, /* do random reads for performance checking */
67     DO_CRASH, /* do multi-threaded crashtest */
68     DO_SPLIT, /* split a field value */
69     DO_STREAM, /* stream in records */
70     DO_MFNLIST, /* fetch records by mfn list */
71     DO_IFLOAD, /* read a lk2-style file from stdin */
72     DO_SWLOAD, /* load stopwords */
73     DO_IFDUMP, /* dump a lk2-style file to stdout */
74     DO_FDT, /* print fdt */
75     DO_VUTF /* validate UTF-8 input */
76     } todo;
77    
78     typedef enum {
79     FMT_MFN, /* rowid only */
80     FMT_MFNF, /* rowid, 1st field */
81     FMT_PROP, /* property style */
82     FMT_TXT, /* plaintext masterfile style */
83     FMT_TXTW /* plaintext masterfile style with W lines */
84     } format;
85    
86     typedef enum {
87     IFM_DUMP,
88     IFM_TAB,
89     IFM_OLD, /* dump old index */
90     IFM_COPY, /* copy old index */
91     IFM_CHK /* check (new oxi) index */
92     } ifmode;
93    
94    
95     /* ************************************************************
96     private data
97     */
98     static const char *pft;
99     static const char **term, **val;
100     static OpenIsisSet *post;
101     int nterm, nval;
102     static int db = -1, wdb = -1, append = 0, idxall = 0;
103    
104     /* ************************************************************
105     private functions
106     */
107     static int argchk ( const char *param, const char *n, const char *v )
108     {
109     return strcmp( param, n ) ? 0 : v ? 1 :
110     (openIsisSMsg( OPENISIS_ERR_INVAL, "no value for param '%s'", param ), 0);
111     } /* argchk */
112    
113    
114     static int print ( OpenIsisRec *r, int freeit, format f )
115     {
116     union { OpenIsisRec r; char buf[10000]; } x;
117     int i, ret;
118     if ( ! r ) {
119     if ( FMT_TXT == f )
120     openIsisSMsg( 1, "\n" );
121     return -1;
122     }
123     ret = r->rowid;
124     if ( pft ) {
125     OpenIsisRec *q;
126     OPENISIS_INITBUF(x);
127     q = openIsisFmt( &x.r, pft, r );
128     if ( freeit )
129     free( r );
130     freeit = q != &x.r;
131     r = q;
132     }
133     if ( FMT_MFN == f )
134     openIsisSMsg( 1, "%d\n", r->rowid );
135     else if ( FMT_MFNF == f )
136     openIsisSMsg( 1, "%d %.*s\n", r->rowid,
137     0 == r->len ? 1 : (int)r->field[0].len,
138     0 == r->len ? "-" : r->field[0].val );
139     else if ( FMT_TXT <= f ) {
140     openIsisSMsg( 1, "\n" ); /* blank line */
141     if ( FMT_TXTW == f )
142     openIsisSMsg( 1, "W\t%d\n", r->rowid );
143     for ( i=0; i<r->len; i++ ) {
144     if ( r->field[i].val )
145     openIsisSMsg( 1, "%d\t%.*s\n", r->field[i].tag,
146     (int)r->field[i].len, r->field[i].val );
147     else
148     openIsisSMsg( 1, "%d\t%d\n", r->field[i].tag, r->field[i].len );
149     }
150     } else for ( i=0; i<r->len; i++ ) {
151     if ( ! r->field[i].val ) { /* shouldn't happen -- numeric ? */
152     openIsisSMsg( 1, "%d.?=%d\n", r->rowid, r->field[i].len );
153     continue;
154     }
155     openIsisSMsg( 1, "%d.%d=%.*s\n", r->rowid, r->field[i].tag,
156     (int)r->field[i].len, r->field[i].val );
157     if ( r->field[i].len && '^' == *r->field[i].val ) { /* split subfields */
158     OpenIsisRec *rf = openIsisReadField( 0, r->field+i );
159     if ( rf ) {
160     int j;
161     for ( j=0; j<rf->len; j++ )
162     openIsisSMsg( 1, "%d.%d.%c=%.*s\n",
163     r->rowid, r->field[i].tag,
164     (0x60 & (int)rf->field[j].tag ) ?
165     (int)rf->field[j].tag : ' ',
166     (int)rf->field[j].len, rf->field[j].val );
167     free( rf );
168     }
169     }
170     }
171     if ( 0 <= wdb ) {
172     OpenIsisRec *q = 0;
173     int ok;
174     if ( append )
175     r->rowid = 0;
176     if ( idxall && r != &x.r ) { /* add index entries for all fields */
177     OPENISIS_INITBUF(x);
178     q = &x.r;
179     for ( i=0; i<r->len; i++ ) {
180     char hit[64];
181     OpenIsisField *fld = r->field + i;
182     sprintf( hit, "%d.%d.%d.1 ", r->rowid, fld->tag, i );
183     OPENISIS_RADDS( q, OPENISIS_XHIT, hit, q != &x.r );
184     OPENISIS_RCAT( q, fld->val, fld->len, q != &x.r );
185     }
186     }
187     ok = openIsisDWritex( wdb, r, q );
188     openIsisSMsg( 1, "wrote mfn %d (%d)\n", r->rowid, ok );
189     }
190     if ( freeit )
191     free( r );
192     return ret;
193     } /* print */
194    
195    
196     static void printid ( int id, format f )
197     {
198     if ( FMT_MFN == f && 0 > wdb )
199     openIsisSMsg( 1, "%d\n", id );
200     else
201     print( openIsisReadRow( db, id ), !0, f );
202     } /* printid */
203    
204    
205     static int printlk2 ( void *me, OpenIsisKey *key, OpenIsisHit *hit )
206     {
207     (void)me;
208     if ( key && hit )
209     /* 30 key BLANK 7 mfn BLANK 5 tag BLANK 4 occ BLANK 4 pos*/
210     openIsisSMsg( 1, "%-30.*s %7u %5u %4u %4u\n",
211     key->len, key->byt, hit->mfn, hit->tag, hit->occ, hit->pos );
212     return 0;
213     } /* printlk2 */
214    
215    
216     static int printtab ( void *me, OpenIsisKey *key, OpenIsisHit *hit )
217     {
218     (void)me;
219     if ( key && hit )
220     /* 30 key BLANK 7 mfn BLANK 5 tag BLANK 4 occ BLANK 4 pos*/
221     openIsisSMsg( 1, "%.*s\t%u\t%u\t%u\t%u\n",
222     key->len, key->byt, hit->mfn, hit->tag, hit->occ, hit->pos );
223     return 0;
224     } /* printtab */
225    
226    
227     /* timing utility. set the timeval, return milliseconds since last call. */
228     #ifdef WIN32
229     static int millis ( struct _timeb *tb )
230     {
231     struct _timeb otb = *tb;
232     _ftime( tb );
233     return (tb->time - otb.time)*1000 + (tb->millitm - otb.millitm);
234     } /* millis */
235     #else
236     static int millis ( struct timeval *tv )
237     {
238     struct timeval otv = *tv;
239     gettimeofday( tv, 0 );
240     return (tv->tv_sec - otv.tv_sec)*1000 + (tv->tv_usec - otv.tv_usec)/1000;
241     } /* millis */
242     #endif
243    
244     #ifdef HAVE_PTHREAD
245     int myOpenIsisLockFunc ( int lock )
246     {
247     static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER; /* the "fast" kind */
248     static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
249     /*
250     LOG_DBG( LOG_ERROR, "thread %d op 0x%08x", (int)pthread_self(), lock );
251     */
252     switch ( OPENISIS_WAIT & lock ) {
253     case OPENISIS_RELE: return pthread_mutex_unlock( &mut );
254     case OPENISIS_LOCK: return pthread_mutex_lock( &mut );
255     case OPENISIS_WAKE: return pthread_cond_broadcast( &cond );
256     case OPENISIS_WAIT: return pthread_cond_wait( &cond, &mut );
257     }
258     return -1;
259     }
260     #else
261     # define myOpenIsisLockFunc 0
262     #endif
263    
264     typedef struct {
265     int start;
266     } threadarg;
267    
268     /* (p)thread routine */
269     static void * run ( void *arg )
270     {
271     struct timeval tv;
272     int m;
273     threadarg *my = (threadarg *)arg;
274     int start = my->start;
275     int i;
276    
277     millis( &tv );
278     for ( i=start+1; i++!=start; ) {
279     int j;
280     if ( i >= nterm )
281     i = 0;
282     for ( j=0; j<post[i].len; j++ ) {
283     int mfn = post[i].id[j];
284     OpenIsisRec *r = openIsisReadRow( db, mfn );
285     if ( ! r ) {
286     openIsisSMsg( 2, "no rec %d\n", mfn );
287     continue;
288     }
289     if ( 0 >= r->len )
290     openIsisSMsg( 2, "no fields for %d\n", mfn );
291     else if ( r->field[0].len != (int)strlen(val[mfn])
292     || memcmp( r->field[0].val, val[mfn], r->field[0].len )
293     )
294     openIsisSMsg( 2, "mismatch on %d\n", mfn );
295     free( r );
296     }
297     }
298     m = millis(&tv);
299     openIsisSMsg( 1, "thread %d@%d terminated after %.3f seconds\n",
300     #ifdef HAVE_THREADS
301     (int)pthread_self()
302     #else
303     0L
304     #endif
305     , start, m/1000. );
306     return (void *)m;
307     } /* run */
308    
309    
310     /* multithreaded crashtest */
311     static int crash ( const char *pre )
312     {
313     struct timeval tv;
314     union { OpenIsisRec r; char buf[10000]; } x;
315     int l;
316     int p = 0;
317     int i, pass;
318     int fd = 2;
319     threadarg def;
320    
321     millis( &tv );
322     x.r.len = 0;
323     x.r.bytes = sizeof(x);
324     nterm = 0;
325     while ( openIsisTerm( &x.r, db, pre ) && x.r.len )
326     nterm += x.r.len;
327     openIsisSMsg( fd, "%d terms\n", nterm );
328     openIsisSMsg( fd, "%.3f sec\n", millis(&tv)/1000. );
329    
330     term = (const char **)malloc( nterm * sizeof(*term) );
331     post = (OpenIsisSet *)malloc( nterm * sizeof(*post) );
332     nterm = 0;
333     nval = 0;
334     while ( openIsisTerm( &x.r, db, pre ) && x.r.len ) {
335     for ( i=0; i<x.r.len; i++, nterm++ ) {
336     int cnt;
337     OpenIsisSet *set = post+nterm;
338     char *c = malloc( x.r.field[i].len + 1 );
339    
340     memcpy( c, x.r.field[i].val, x.r.field[i].len );
341     c[ x.r.field[i].len ] = 0;
342     term[ nterm ] = c;
343    
344     set->len = 0;
345     openIsisQuery( set, db, c, OPENISIS_QRY_KEYEQ, 0 );
346     if ( 0 >= set->len ) {
347     openIsisSMsg( 2, "no results for '%s'\n", c );
348     return 1;
349     }
350     for ( cnt = 0; cnt < set->len; cnt++ )
351     if ( nval < set->id[cnt] )
352     nval = set->id[cnt];
353     p += set->len;
354     }
355     }
356     openIsisSMsg( fd, "%d postings max mfn %d\n", p, nval );
357     openIsisSMsg( fd, "%.3f sec\n", millis(&tv)/1000. );
358    
359     val = (const char **)malloc( (1+nval) * sizeof(*val) );
360     for ( i=1; i<=nval; i++ ) {
361     OpenIsisRec *r = openIsisReadRow( db, i );
362     val[i] = 0;
363     if ( ! r ) {
364     openIsisSMsg( 2, "no rec %d\n", i );
365     continue;
366     }
367     if ( 0 < r->len ) {
368     char *c = malloc( r->field[0].len + 1 );
369    
370     memcpy( c, r->field[0].val, r->field[0].len );
371     c[ r->field[0].len ] = 0;
372     val[i] = c;
373     }
374     free( r );
375     }
376     l = millis(&tv);
377     openIsisSMsg( fd, "sequential read %d rows in %.3f seconds %d rows per sec\n",
378     nval, l/1000., nval*1000/(l?l:1) );
379    
380     def.start = 0;
381     i = 0;
382     run( &def );
383     l = millis(&tv);
384     run( &def );
385     l = millis(&tv);
386     run( &def );
387     l = millis(&tv);
388     openIsisSMsg( fd, "in-thread run in %.3f seconds %d rows per sec\n",
389     l/1000., p*1000/(l?l:1) );
390     #ifdef HAVE_THREADS
391     #define passes 3
392     {
393     static int nThreads[] = { 8, 2, 1, 4 };
394     int res[ sizeof(nThreads)/sizeof(nThreads[0]) ];
395     pthread_t th[ 8 /* max. nThreads */ ];
396     threadarg arg[ sizeof(th)/sizeof(th[0]) ];
397     int j;
398    
399    
400     for ( i=0; i<(int)(sizeof(nThreads)/sizeof(nThreads[0])); i++ )
401     res[i] = 0;
402     for ( j=0; j<(int)(sizeof(th)/sizeof(th[0])); j++ )
403     #if 0
404     if ( ! (arg[j].ses = openIsisSesGet( -1, 0 )) ) {
405     openIsisSMsg( fd, "could not get %dth session\n", j );
406     return 1;
407     }
408     #endif
409     for ( pass=0; pass<passes; pass++ )
410     for ( i=0; i<(int)(sizeof(nThreads)/sizeof(nThreads[0])); i++ ) {
411     int rps, avg;
412     millis(&tv);
413     assert( nThreads[i] <= (int)(sizeof(th)/sizeof(th[0])) );
414     for ( j=0; j<nThreads[i]; j++ ) {
415     arg[j].start = j*nterm/nThreads[i];
416     if ( pthread_create( th+j, 0, run, arg+j ) )
417     th[j] = (pthread_t)0;
418     }
419     openIsisSMsg( fd, "started %d threads\n", nThreads[i] );
420     avg = 0;
421     for ( j=0; j<nThreads[i]; j++ ) {
422     int t;
423     pthread_join( th[j], (void**)&t );
424     avg += t;
425     }
426     avg /= nThreads[i];
427     l = millis(&tv);
428     rps = (int)(nThreads[i] * p * 1000 / avg);
429     openIsisSMsg( fd, "joined %d threads avg %.3f after %.3f seconds %d rows per sec\n",
430     nThreads[i], avg/1000., l/1000., rps );
431     res[i] += rps;
432     }
433     for ( i=0; i<(int)(sizeof(nThreads)/sizeof(nThreads[0])); i++ )
434     openIsisSMsg( fd, "%d threads %d rows per sec\n", nThreads[i], res[i]/passes );
435     }
436     #else
437     (void)pass; /* avoid compiler warning */
438     #endif /* HAVE_THREADS */
439     return 0;
440     } /* crash */
441    
442    
443     /* ************************************************************
444     package functions
445     */
446     /* ************************************************************
447     public functions
448     */
449     int main ( int argc, const char **argv )
450     {
451     int ret = 0;
452     int i,intarg=0;
453     todo what = DO_DUMP;
454     format fmt = FMT_TXT;
455     int check = OPENISIS_CHK_FIX;
456     int searchmode = OPENISIS_QRY_KEYAT;
457     int idxmode = OPENISIS_IDXPF;
458     const char *search = 0;
459     const char *idxto = 0;
460     int needdb = !0;
461     const char *dowrite = 0;
462     ifmode ifm = IFM_DUMP;
463     OpenIsisRec *argr;
464     OpenIsisDb *odb;
465    
466     /* initialize minimal env */
467     openIsisCOpen(0);
468     argr = openIsisRSet( 0,
469     OPENISIS_RFDT|OPENISIS_RARGV|OPENISIS_RIGN | (argc-1),
470     openIsisFdtSyspar, argv+1 );
471    
472     if ( 2 == argc && ! strcmp("-version",argv[1]) ) {
473     openIsisSMsg( 0, "%s\n", OPENISIS_VERSION );
474     goto bye;
475     }
476     /* check options ... */
477     for ( i=0; i < argc; ) {
478     const char *n = argv[i], *v = 0;
479     assert( n );
480     if ( '-' == n[0] )
481     n++;
482     if ( 1 == argc - i || '-' == argv[i+1][0] ) { /* no value */
483     i++;
484     } else {
485     v = argv[i+1];
486     assert( v );
487     i += 2;
488     }
489    
490     if ( argchk("logfile",n,v) )
491     openIsisLog( '=', v );
492     else if ( argchk("v",n,v) )
493     openIsisLog( *v, 0 );
494     else if ( argchk("scan",n,v) ) {
495     what = DO_SCAN;
496     search = v;
497     }
498     else if ( argchk("search",n,v) ) {
499     what = DO_SEARCH;
500     search = v;
501     }
502     else if ( argchk("upto",n,v) ) {
503     idxmode = OPENISIS_IDXUPTO;
504     idxto = v;
505     }
506     else if ( argchk("incl",n,v) ) {
507     idxmode = OPENISIS_IDXINCL;
508     idxto = v;
509     }
510     else if ( argchk("query",n,v) ) {
511     what = DO_SEARCH;
512     searchmode = OPENISIS_QRY_SIMPLE;
513     search = v;
514     }
515     else if ( argchk("terms",n,v) ) {
516     what = DO_TERMS;
517     search = v;
518     }
519     else if ( argchk("perf",n,v) ) {
520     what = DO_PERF;
521     intarg = atoi(v);
522     }
523     else if ( argchk("crash",n,v) ) {
524     what = DO_CRASH;
525     search = v;
526     }
527     else if ( argchk("split",n,v) ) {
528     what = DO_SPLIT;
529     search = v;
530     needdb = 0;
531     }
532     else if ( argchk("fmt",n,v) ) {
533     if ( ! strcmp("mfn",v) )
534     fmt =FMT_MFN;
535     else if ( ! strcmp("mfnf",v) )
536     fmt =FMT_MFNF;
537     else if ( ! strcmp("prop",v) )
538     fmt =FMT_PROP;
539     else if ( ! strcmp("txt",v) )
540     fmt =FMT_TXT;
541     else if ( ! strcmp("txtw",v) )
542     fmt =FMT_TXTW;
543     }
544     else if ( ! strcmp("check",n) )
545     what = DO_CHECK;
546     else if ( ! strcmp("vutf",n) ) {
547     what = DO_VUTF;
548     needdb = 0;
549     } else if ( argchk("pft",n,v) )
550     pft = v;
551     else if ( ! strcmp("stream",n) ) {
552     what = DO_STREAM;
553     needdb = 0;
554     } else if ( argchk("write",n,v) )
555     dowrite = v;
556     else if ( argchk("append",n,v) ) {
557     dowrite = v;
558     append = !0;
559     } else if ( ! strcmp("idxall",n) )
560     idxall = !0;
561     else if ( ! strcmp("mfnlist",n) )
562     what = DO_MFNLIST;
563     else if ( argchk("ifload",n,v) ) {
564     what = DO_IFLOAD;
565     intarg = atoi(v);
566     } else if ( ! strcmp("swload",n) ) {
567     what = DO_SWLOAD;
568     } else if ( ! strcmp("ifadd",n) ) {
569     what = DO_IFLOAD;
570     intarg = -1;
571     } else if ( ! strcmp("ifdel",n) ) {
572     what = DO_IFLOAD;
573     intarg = -2;
574     } else if ( ! strcmp("ifcopy",n) ) {
575     what = DO_IFDUMP;
576     ifm = IFM_COPY;
577     } else if ( ! strcmp("ifchk",n) ) {
578     what = DO_IFDUMP;
579     ifm = IFM_CHK;
580     } else if ( ! strcmp("iftab",n) ) {
581     what = DO_IFDUMP;
582     ifm = IFM_TAB;
583     } else if ( ! strcmp("ifdump",n) )
584     what = DO_IFDUMP;
585     else if ( ! strcmp("noxi",n) )
586     ifm = IFM_OLD;
587     else if ( argchk("out",n,v) ) {
588     char buf[256] = ">";
589     int l = strlen(v);
590     if ( l < 254 ) {
591     memcpy( buf+1, v, l );
592     buf[l+2] = 0;
593     openIsisSOpen( buf, 0, 0 );
594     }
595     }
596     else if ( argchk("fdtdump",n,v) ) {
597     what = DO_FDT;
598     intarg = atoi(v);
599     }
600     } /* while argc */
601    
602    
603     if ( needdb && 0 > (db = openIsisOpen( 0, argv + 1, argc - 1 )) ) {
604     openIsisSMsg( 2,
605     "openisis " OPENISIS_VERSION "\n\n"
606     "please specify a valid database with -db, e.g.\n"
607     "-db /winisis/data/cds\n"
608     "\n"
609     "other options are:\n"
610     "-search term search for term\n"
611     "-query \"query\" run a query like \"water * plant\"\n"
612     "-terms term list terms matching term (e.g. plant$)\n"
613     );
614     /* warning: string length `580' is greater than the minimum length
615     * `509' ISO C89 is required to support
616     */
617     openIsisSMsg( 2,
618     "-fmt mfn for a search or query, list only the mfn\n"
619     "-fmt mfnf for a search or query, list the mfn and 1st field\n"
620     "-pft \"pft\" use printformat (currently very limited)\n"
621     "-write dbpath specify a db where records are written to\n"
622     "-mfnlist read mfns from stdin\n"
623     "-ifload pctfree read .lk2-index from stdin\n"
624     "\n"
625     "default output format is one field per line like tag<TAB>value\n"
626     );
627     ret = 1;
628     goto bye;
629     }
630    
631     if ( dowrite && 0 > (wdb = openIsisOpen( dowrite, 0, 0 )) ) {
632     openIsisSMsg( 2, "could not open write target db '%s'\n", dowrite );
633     ret = 2;
634     goto bye;
635     }
636    
637    
638     switch ( what ) {
639     case DO_DUMP: {
640     int max = openIsisMaxRowid( db );
641     int rowid;
642     for ( rowid = 1; rowid <= max; rowid++ )
643     printid( rowid, fmt );
644     } break; /* DO_DUMP */
645     case DO_MFNLIST: {
646     char *buf = 0;
647     int l;
648     while ( 0 <= (l = openIsisSReadln( &buf )) ) {
649     int id = 0;
650     while ( l-- )
651     id = 10*id + *buf++ - '0';
652     if ( id )
653     printid( id, fmt );
654     }
655     } break; /* DO_MFNLIST */
656     case DO_SCAN: {
657     int max = openIsisMaxRowid( db );
658     int rowid;
659     for ( rowid = 1; 0 < rowid && rowid <= max; rowid++ )
660     rowid = print( openIsisScan( db, rowid, 0, search ), !0, fmt );
661     } break; /* DO_SCAN */
662     case DO_SEARCH: {
663     int cnt;
664     OpenIsisSet set;
665     set.len = 0;
666     openIsisQuery( &set, db, search, searchmode, 0 );
667     if ( 0 >= set.len ) {
668     openIsisSMsg( 2, "no results for '%s'\n", search );
669     ret = 1;
670     goto bye;
671     }
672     /* openIsisSMsg( 2, "%d\trows for\t%s\n", set.len, search ); */
673     for ( cnt = 0; cnt < set.len; cnt++ )
674     printid( set.id[cnt], fmt );
675     } break; /* DO_SEARCH */
676     case DO_TERMS: {
677     union { OpenIsisRec r; char buf[10000]; } x;
678     x.r.len = 0;
679     x.r.bytes = sizeof(x);
680     while ( openIsisTerm( &x.r, db, search ) && x.r.len ) {
681     /* openIsisSMsg( 1, "%d terms\n", x.r.len ); */
682     for ( i=0; i<x.r.len; i++ )
683     openIsisSMsg( 1, "%.*s\n", (int)x.r.field[i].len, x.r.field[i].val );
684     }
685     } break; /* DO_TERMS */
686     case DO_PERF: {
687     int max = openIsisMaxRowid( db );
688     while ( 0 < intarg-- ) {
689     OpenIsisRec *r = openIsisReadRow( db, 1+((int)rand() % max) );
690     free( r );
691     }
692     } break; /* DO_PERF */
693     case DO_CHECK:
694     ret = openIsisCheck( db, check );
695     goto bye;
696     case DO_CRASH:
697     ret = crash( search );
698     goto bye;
699     case DO_SPLIT: {
700     OpenIsisField f;
701     OpenIsisRec *r;
702     f.tag = 24; f.val = search; f.len = strlen(search);
703     r = openIsisReadField( 0, &f );
704     if ( r )
705     for ( i=0; i<r->len; i++ )
706     openIsisSMsg( 1, "%c=%.*s\n", (int)r->field[i].tag,
707     (int)r->field[i].len, r->field[i].val );
708     } break;
709     case DO_STREAM: {
710     OpenIsisIos ios;
711     OpenIsisRecStream rs = { 0, OPENISIS_STOPONEMPTY, 0, 0, 0 };
712     LIO_SINIT( &ios, lio_stdio, "stdin", LIO_IN );
713     rs.in = &ios; /* some gcc versions need it this way */
714     while ( 0 < (i = openIsisSGetr( &rs )) )
715     print( rs.rec, 0, fmt );
716     } break;
717     case DO_IFLOAD:
718     case DO_SWLOAD: {
719     OpenIsisKey key;
720     OpenIsisHit hit;
721     OpenIsisIndex idx = openIsisIdxOpen( db, intarg );
722     char *buf = 0;
723     int l, lines = 0;
724    
725     memset( &hit, 0, sizeof(hit) );
726     hit.dbn = (-2 == intarg) ? 0xffff : 0; /* secret key for ifdel */
727     while ( 0 <= (l = openIsisSReadln( &buf )) && buf ) {
728     char *t = memchr( buf, '\t', l );
729     if ( DO_SWLOAD == what ) {
730     memcpy( key.byt, buf, key.len = (unsigned char)l );
731     } else if ( t ) { /* tab delimited */
732     key.len = (unsigned char)(t - buf);
733     memcpy( key.byt, buf, key.len );
734     if ( 0 >= (l -= t-buf+1) || !(t = memchr( buf=t+1, '\t', l )) )
735     continue;
736     hit.mfn = (unsigned) openIsisA2i( buf, t-buf );
737     if ( 0 >= (l -= t-buf+1) || !(t = memchr( buf=t+1, '\t', l )) )
738     continue;
739     hit.tag = (unsigned short) openIsisA2i( buf, t-buf );
740     if ( 0 >= (l -= t-buf+1) || !(t = memchr( buf=t+1, '\t', l )) )
741     continue;
742     hit.occ = (unsigned short) openIsisA2i( buf, t-buf );
743     if ( 0 >= (l -= t-buf+1) )
744     continue;
745     hit.pos = (unsigned short) openIsisA2i( t+1, l );
746     } else {
747     /* 10/30 key BLANK 7 mfn BLANK 5 tag BLANK 4 occ BLANK 4 pos*/
748     int eok = l - 24; /* pos of blank after key, 10 or 30 */
749     if ( 54 != l && 34 != l ) {
750     openIsisSMsg( OPENISIS_ERR_INVAL,
751     "bad ifload input len %d, want 34 or54 bytes + newline\n", l );
752     break;
753     }
754     for ( i=eok-1; ' ' == buf[i] && i--; )
755     ;
756     key.len = (unsigned char) (++i);
757     memcpy( key.byt, buf, key.len );
758     log_msg( LOG_VERBOSE, "'%.*s'", 7, buf+eok+1 );
759     hit.mfn = (unsigned) openIsisA2i( buf+eok+1, 7 );
760     hit.tag = (unsigned short)openIsisA2i( buf+eok+9, 5 );
761     hit.occ = (unsigned short)openIsisA2i( buf+eok+15, 4 );
762     hit.pos = (unsigned short)openIsisA2i( buf+eok+20, 4 );
763     }
764     log_msg( LOG_VERBOSE, "'%.*s' %d %d %d %d",
765     key.len, key.byt, hit.mfn, hit.tag, hit.occ, hit.pos );
766     if ( openIsisIdxAdd( idx, &key, &hit ) )
767     break;
768     if ( !(++lines & 0x3ff) )
769     log_msg( LOG_INFO, "%dK lines", lines >> 10 );
770     }
771     openIsisIdxDone( idx );
772     } break;
773     case DO_IFDUMP: {
774     OpenIsisIdxLoop l;
775     memset( &l, 0, sizeof(l) );
776     l.flg = idxmode;
777     switch ( ifm ) {
778     case IFM_OLD:
779     l.flg |= OPENISIS_IDXTRAD;
780     case IFM_DUMP:
781     l.cb = (OpenIsisIdxCb*)printlk2;
782     break;
783     case IFM_TAB:
784     l.cb = (OpenIsisIdxCb*)printtab;
785     break;
786     case IFM_COPY:
787     l.flg |= OPENISIS_IDXTRAD;
788     l.me = openIsisIdxOpen( 0 <= wdb ? wdb : db, 0 );
789     l.cb = (OpenIsisIdxCb*)openIsisIdxAdd;
790     break;
791     case IFM_CHK:
792     /* nuttin */
793     break;
794     }
795     if ( search )
796     memcpy( l.key.byt, search,
797     l.key.len = (unsigned char)strlen( search ) );
798     if ( idxto )
799     memcpy( l.to.byt, idxto,
800     l.to.len = (unsigned char)strlen( idxto ) );
801     openIsisIdxLoop( db, &l );
802     if ( IFM_COPY == ifm )
803     openIsisIdxDone( (OpenIsisIndex)l.me );
804     } break;
805     case DO_FDT:
806     odb = ldb_getdb( db );
807     if ( odb && odb->fdt )
808     print( openIsisFFdt2Rec( odb->fdt, 0, intarg ), 0, fmt );
809     break;
810     case DO_VUTF: {
811     char buf[1024];
812     int t = 0, f = 0, g;
813     while ( 0 < (g = lio_read( &lio_in, buf, sizeof(buf) )) ) {
814     int l = openIsisValidUTF8( buf, g, &f );
815     if ( l ) {
816     openIsisSMsg( OPENISIS_ERR_INVAL,
817     "at total %d = %d+%d\n", l-1+t, l-1, t );
818     ret = 1;
819     goto bye;
820     }
821     t += g;
822     }
823     }
824     } /* switch ( what ) */
825     bye:
826     if ( 0 <= db )
827     openIsisClose( db );
828     if ( 0 <= wdb )
829     openIsisClose( wdb );
830    
831     /* at least with WINE,
832     atexit cleanup is not performed
833     unless we explicitly call exit :(
834     */
835     exit( ret );
836     return ret;
837     } /* openisis main */

  ViewVC Help
Powered by ViewVC 1.1.26