1 |
dpavlin |
237 |
/* |
2 |
|
|
openisis - an open implementation of the CDS/ISIS database |
3 |
|
|
Version 0.8.x (patchlevel see file Version) |
4 |
|
|
Copyright (C) 2001-2003 by Erik Grziwotz, erik@openisis.org |
5 |
|
|
|
6 |
|
|
This library is free software; you can redistribute it and/or |
7 |
|
|
modify it under the terms of the GNU Lesser General Public |
8 |
|
|
License as published by the Free Software Foundation; either |
9 |
|
|
version 2.1 of the License, or (at your option) any later version. |
10 |
|
|
|
11 |
|
|
This library is distributed in the hope that it will be useful, |
12 |
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of |
13 |
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
14 |
|
|
Lesser General Public License for more details. |
15 |
|
|
|
16 |
|
|
You should have received a copy of the GNU Lesser General Public |
17 |
|
|
License along with this library; if not, write to the Free Software |
18 |
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
19 |
|
|
|
20 |
|
|
see README for more information |
21 |
|
|
EOH */ |
22 |
|
|
|
23 |
|
|
/* |
24 |
|
|
$Id: openisis.c,v 1.67 2003/06/03 11:25:02 kripke Exp $ |
25 |
|
|
main file of openisis executable. |
26 |
|
|
*/ |
27 |
|
|
|
28 |
|
|
#ifdef HAVE_PTHREAD |
29 |
|
|
#define HAVE_THREADS |
30 |
|
|
#endif |
31 |
|
|
|
32 |
|
|
#include <assert.h> |
33 |
|
|
#include <stdlib.h> /* free */ |
34 |
|
|
#include <string.h> /* strcmp */ |
35 |
|
|
#ifdef WIN32 |
36 |
|
|
# include <sys/timeb.h> |
37 |
|
|
# include <sys/types.h> |
38 |
|
|
# define timeval _timeb |
39 |
|
|
#else |
40 |
|
|
# include <unistd.h> /* gettimeofday */ |
41 |
|
|
#endif |
42 |
|
|
#include <sys/time.h> /* gettimeofday */ |
43 |
|
|
#ifdef HAVE_PTHREAD |
44 |
|
|
#include <pthread.h> /* threaded crashtest */ |
45 |
|
|
#endif |
46 |
|
|
|
47 |
|
|
#include "openisis.h" |
48 |
|
|
|
49 |
|
|
/* |
50 |
|
|
temporary library includes for testing of functions |
51 |
|
|
that should later be accessible via openisis.h |
52 |
|
|
*/ |
53 |
|
|
#include "lio.h" |
54 |
|
|
extern Db* ldb_getdb (int dbid); |
55 |
|
|
|
56 |
|
|
|
57 |
|
|
/* ************************************************************ |
58 |
|
|
private types |
59 |
|
|
*/ |
60 |
|
|
typedef enum { /* what to do */ |
61 |
|
|
DO_DUMP, |
62 |
|
|
DO_SCAN, /* simple full scan */ |
63 |
|
|
DO_SEARCH, /* basic index based searching */ |
64 |
|
|
DO_TERMS, /* list terms */ |
65 |
|
|
DO_CHECK, /* check db */ |
66 |
|
|
DO_PERF, /* do random reads for performance checking */ |
67 |
|
|
DO_CRASH, /* do multi-threaded crashtest */ |
68 |
|
|
DO_SPLIT, /* split a field value */ |
69 |
|
|
DO_STREAM, /* stream in records */ |
70 |
|
|
DO_MFNLIST, /* fetch records by mfn list */ |
71 |
|
|
DO_IFLOAD, /* read a lk2-style file from stdin */ |
72 |
|
|
DO_SWLOAD, /* load stopwords */ |
73 |
|
|
DO_IFDUMP, /* dump a lk2-style file to stdout */ |
74 |
|
|
DO_FDT, /* print fdt */ |
75 |
|
|
DO_VUTF /* validate UTF-8 input */ |
76 |
|
|
} todo; |
77 |
|
|
|
78 |
|
|
typedef enum { |
79 |
|
|
FMT_MFN, /* rowid only */ |
80 |
|
|
FMT_MFNF, /* rowid, 1st field */ |
81 |
|
|
FMT_PROP, /* property style */ |
82 |
|
|
FMT_TXT, /* plaintext masterfile style */ |
83 |
|
|
FMT_TXTW /* plaintext masterfile style with W lines */ |
84 |
|
|
} format; |
85 |
|
|
|
86 |
|
|
typedef enum { |
87 |
|
|
IFM_DUMP, |
88 |
|
|
IFM_TAB, |
89 |
|
|
IFM_OLD, /* dump old index */ |
90 |
|
|
IFM_COPY, /* copy old index */ |
91 |
|
|
IFM_CHK /* check (new oxi) index */ |
92 |
|
|
} ifmode; |
93 |
|
|
|
94 |
|
|
|
95 |
|
|
/* ************************************************************ |
96 |
|
|
private data |
97 |
|
|
*/ |
98 |
|
|
static const char *pft; |
99 |
|
|
static const char **term, **val; |
100 |
|
|
static OpenIsisSet *post; |
101 |
|
|
int nterm, nval; |
102 |
|
|
static int db = -1, wdb = -1, append = 0, idxall = 0; |
103 |
|
|
|
104 |
|
|
/* ************************************************************ |
105 |
|
|
private functions |
106 |
|
|
*/ |
107 |
|
|
static int argchk ( const char *param, const char *n, const char *v ) |
108 |
|
|
{ |
109 |
|
|
return strcmp( param, n ) ? 0 : v ? 1 : |
110 |
|
|
(openIsisSMsg( OPENISIS_ERR_INVAL, "no value for param '%s'", param ), 0); |
111 |
|
|
} /* argchk */ |
112 |
|
|
|
113 |
|
|
|
114 |
|
|
static int print ( OpenIsisRec *r, int freeit, format f ) |
115 |
|
|
{ |
116 |
|
|
union { OpenIsisRec r; char buf[10000]; } x; |
117 |
|
|
int i, ret; |
118 |
|
|
if ( ! r ) { |
119 |
|
|
if ( FMT_TXT == f ) |
120 |
|
|
openIsisSMsg( 1, "\n" ); |
121 |
|
|
return -1; |
122 |
|
|
} |
123 |
|
|
ret = r->rowid; |
124 |
|
|
if ( pft ) { |
125 |
|
|
OpenIsisRec *q; |
126 |
|
|
OPENISIS_INITBUF(x); |
127 |
|
|
q = openIsisFmt( &x.r, pft, r ); |
128 |
|
|
if ( freeit ) |
129 |
|
|
free( r ); |
130 |
|
|
freeit = q != &x.r; |
131 |
|
|
r = q; |
132 |
|
|
} |
133 |
|
|
if ( FMT_MFN == f ) |
134 |
|
|
openIsisSMsg( 1, "%d\n", r->rowid ); |
135 |
|
|
else if ( FMT_MFNF == f ) |
136 |
|
|
openIsisSMsg( 1, "%d %.*s\n", r->rowid, |
137 |
|
|
0 == r->len ? 1 : (int)r->field[0].len, |
138 |
|
|
0 == r->len ? "-" : r->field[0].val ); |
139 |
|
|
else if ( FMT_TXT <= f ) { |
140 |
|
|
openIsisSMsg( 1, "\n" ); /* blank line */ |
141 |
|
|
if ( FMT_TXTW == f ) |
142 |
|
|
openIsisSMsg( 1, "W\t%d\n", r->rowid ); |
143 |
|
|
for ( i=0; i<r->len; i++ ) { |
144 |
|
|
if ( r->field[i].val ) |
145 |
|
|
openIsisSMsg( 1, "%d\t%.*s\n", r->field[i].tag, |
146 |
|
|
(int)r->field[i].len, r->field[i].val ); |
147 |
|
|
else |
148 |
|
|
openIsisSMsg( 1, "%d\t%d\n", r->field[i].tag, r->field[i].len ); |
149 |
|
|
} |
150 |
|
|
} else for ( i=0; i<r->len; i++ ) { |
151 |
|
|
if ( ! r->field[i].val ) { /* shouldn't happen -- numeric ? */ |
152 |
|
|
openIsisSMsg( 1, "%d.?=%d\n", r->rowid, r->field[i].len ); |
153 |
|
|
continue; |
154 |
|
|
} |
155 |
|
|
openIsisSMsg( 1, "%d.%d=%.*s\n", r->rowid, r->field[i].tag, |
156 |
|
|
(int)r->field[i].len, r->field[i].val ); |
157 |
|
|
if ( r->field[i].len && '^' == *r->field[i].val ) { /* split subfields */ |
158 |
|
|
OpenIsisRec *rf = openIsisReadField( 0, r->field+i ); |
159 |
|
|
if ( rf ) { |
160 |
|
|
int j; |
161 |
|
|
for ( j=0; j<rf->len; j++ ) |
162 |
|
|
openIsisSMsg( 1, "%d.%d.%c=%.*s\n", |
163 |
|
|
r->rowid, r->field[i].tag, |
164 |
|
|
(0x60 & (int)rf->field[j].tag ) ? |
165 |
|
|
(int)rf->field[j].tag : ' ', |
166 |
|
|
(int)rf->field[j].len, rf->field[j].val ); |
167 |
|
|
free( rf ); |
168 |
|
|
} |
169 |
|
|
} |
170 |
|
|
} |
171 |
|
|
if ( 0 <= wdb ) { |
172 |
|
|
OpenIsisRec *q = 0; |
173 |
|
|
int ok; |
174 |
|
|
if ( append ) |
175 |
|
|
r->rowid = 0; |
176 |
|
|
if ( idxall && r != &x.r ) { /* add index entries for all fields */ |
177 |
|
|
OPENISIS_INITBUF(x); |
178 |
|
|
q = &x.r; |
179 |
|
|
for ( i=0; i<r->len; i++ ) { |
180 |
|
|
char hit[64]; |
181 |
|
|
OpenIsisField *fld = r->field + i; |
182 |
|
|
sprintf( hit, "%d.%d.%d.1 ", r->rowid, fld->tag, i ); |
183 |
|
|
OPENISIS_RADDS( q, OPENISIS_XHIT, hit, q != &x.r ); |
184 |
|
|
OPENISIS_RCAT( q, fld->val, fld->len, q != &x.r ); |
185 |
|
|
} |
186 |
|
|
} |
187 |
|
|
ok = openIsisDWritex( wdb, r, q ); |
188 |
|
|
openIsisSMsg( 1, "wrote mfn %d (%d)\n", r->rowid, ok ); |
189 |
|
|
} |
190 |
|
|
if ( freeit ) |
191 |
|
|
free( r ); |
192 |
|
|
return ret; |
193 |
|
|
} /* print */ |
194 |
|
|
|
195 |
|
|
|
196 |
|
|
static void printid ( int id, format f ) |
197 |
|
|
{ |
198 |
|
|
if ( FMT_MFN == f && 0 > wdb ) |
199 |
|
|
openIsisSMsg( 1, "%d\n", id ); |
200 |
|
|
else |
201 |
|
|
print( openIsisReadRow( db, id ), !0, f ); |
202 |
|
|
} /* printid */ |
203 |
|
|
|
204 |
|
|
|
205 |
|
|
static int printlk2 ( void *me, OpenIsisKey *key, OpenIsisHit *hit ) |
206 |
|
|
{ |
207 |
|
|
(void)me; |
208 |
|
|
if ( key && hit ) |
209 |
|
|
/* 30 key BLANK 7 mfn BLANK 5 tag BLANK 4 occ BLANK 4 pos*/ |
210 |
|
|
openIsisSMsg( 1, "%-30.*s %7u %5u %4u %4u\n", |
211 |
|
|
key->len, key->byt, hit->mfn, hit->tag, hit->occ, hit->pos ); |
212 |
|
|
return 0; |
213 |
|
|
} /* printlk2 */ |
214 |
|
|
|
215 |
|
|
|
216 |
|
|
static int printtab ( void *me, OpenIsisKey *key, OpenIsisHit *hit ) |
217 |
|
|
{ |
218 |
|
|
(void)me; |
219 |
|
|
if ( key && hit ) |
220 |
|
|
/* 30 key BLANK 7 mfn BLANK 5 tag BLANK 4 occ BLANK 4 pos*/ |
221 |
|
|
openIsisSMsg( 1, "%.*s\t%u\t%u\t%u\t%u\n", |
222 |
|
|
key->len, key->byt, hit->mfn, hit->tag, hit->occ, hit->pos ); |
223 |
|
|
return 0; |
224 |
|
|
} /* printtab */ |
225 |
|
|
|
226 |
|
|
|
227 |
|
|
/* timing utility. set the timeval, return milliseconds since last call. */ |
228 |
|
|
#ifdef WIN32 |
229 |
|
|
static int millis ( struct _timeb *tb ) |
230 |
|
|
{ |
231 |
|
|
struct _timeb otb = *tb; |
232 |
|
|
_ftime( tb ); |
233 |
|
|
return (tb->time - otb.time)*1000 + (tb->millitm - otb.millitm); |
234 |
|
|
} /* millis */ |
235 |
|
|
#else |
236 |
|
|
static int millis ( struct timeval *tv ) |
237 |
|
|
{ |
238 |
|
|
struct timeval otv = *tv; |
239 |
|
|
gettimeofday( tv, 0 ); |
240 |
|
|
return (tv->tv_sec - otv.tv_sec)*1000 + (tv->tv_usec - otv.tv_usec)/1000; |
241 |
|
|
} /* millis */ |
242 |
|
|
#endif |
243 |
|
|
|
244 |
|
|
#ifdef HAVE_PTHREAD |
245 |
|
|
int myOpenIsisLockFunc ( int lock ) |
246 |
|
|
{ |
247 |
|
|
static pthread_mutex_t mut = PTHREAD_MUTEX_INITIALIZER; /* the "fast" kind */ |
248 |
|
|
static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; |
249 |
|
|
/* |
250 |
|
|
LOG_DBG( LOG_ERROR, "thread %d op 0x%08x", (int)pthread_self(), lock ); |
251 |
|
|
*/ |
252 |
|
|
switch ( OPENISIS_WAIT & lock ) { |
253 |
|
|
case OPENISIS_RELE: return pthread_mutex_unlock( &mut ); |
254 |
|
|
case OPENISIS_LOCK: return pthread_mutex_lock( &mut ); |
255 |
|
|
case OPENISIS_WAKE: return pthread_cond_broadcast( &cond ); |
256 |
|
|
case OPENISIS_WAIT: return pthread_cond_wait( &cond, &mut ); |
257 |
|
|
} |
258 |
|
|
return -1; |
259 |
|
|
} |
260 |
|
|
#else |
261 |
|
|
# define myOpenIsisLockFunc 0 |
262 |
|
|
#endif |
263 |
|
|
|
264 |
|
|
typedef struct { |
265 |
|
|
int start; |
266 |
|
|
} threadarg; |
267 |
|
|
|
268 |
|
|
/* (p)thread routine */ |
269 |
|
|
static void * run ( void *arg ) |
270 |
|
|
{ |
271 |
|
|
struct timeval tv; |
272 |
|
|
int m; |
273 |
|
|
threadarg *my = (threadarg *)arg; |
274 |
|
|
int start = my->start; |
275 |
|
|
int i; |
276 |
|
|
|
277 |
|
|
millis( &tv ); |
278 |
|
|
for ( i=start+1; i++!=start; ) { |
279 |
|
|
int j; |
280 |
|
|
if ( i >= nterm ) |
281 |
|
|
i = 0; |
282 |
|
|
for ( j=0; j<post[i].len; j++ ) { |
283 |
|
|
int mfn = post[i].id[j]; |
284 |
|
|
OpenIsisRec *r = openIsisReadRow( db, mfn ); |
285 |
|
|
if ( ! r ) { |
286 |
|
|
openIsisSMsg( 2, "no rec %d\n", mfn ); |
287 |
|
|
continue; |
288 |
|
|
} |
289 |
|
|
if ( 0 >= r->len ) |
290 |
|
|
openIsisSMsg( 2, "no fields for %d\n", mfn ); |
291 |
|
|
else if ( r->field[0].len != (int)strlen(val[mfn]) |
292 |
|
|
|| memcmp( r->field[0].val, val[mfn], r->field[0].len ) |
293 |
|
|
) |
294 |
|
|
openIsisSMsg( 2, "mismatch on %d\n", mfn ); |
295 |
|
|
free( r ); |
296 |
|
|
} |
297 |
|
|
} |
298 |
|
|
m = millis(&tv); |
299 |
|
|
openIsisSMsg( 1, "thread %d@%d terminated after %.3f seconds\n", |
300 |
|
|
#ifdef HAVE_THREADS |
301 |
|
|
(int)pthread_self() |
302 |
|
|
#else |
303 |
|
|
0L |
304 |
|
|
#endif |
305 |
|
|
, start, m/1000. ); |
306 |
|
|
return (void *)m; |
307 |
|
|
} /* run */ |
308 |
|
|
|
309 |
|
|
|
310 |
|
|
/* multithreaded crashtest */ |
311 |
|
|
static int crash ( const char *pre ) |
312 |
|
|
{ |
313 |
|
|
struct timeval tv; |
314 |
|
|
union { OpenIsisRec r; char buf[10000]; } x; |
315 |
|
|
int l; |
316 |
|
|
int p = 0; |
317 |
|
|
int i, pass; |
318 |
|
|
int fd = 2; |
319 |
|
|
threadarg def; |
320 |
|
|
|
321 |
|
|
millis( &tv ); |
322 |
|
|
x.r.len = 0; |
323 |
|
|
x.r.bytes = sizeof(x); |
324 |
|
|
nterm = 0; |
325 |
|
|
while ( openIsisTerm( &x.r, db, pre ) && x.r.len ) |
326 |
|
|
nterm += x.r.len; |
327 |
|
|
openIsisSMsg( fd, "%d terms\n", nterm ); |
328 |
|
|
openIsisSMsg( fd, "%.3f sec\n", millis(&tv)/1000. ); |
329 |
|
|
|
330 |
|
|
term = (const char **)malloc( nterm * sizeof(*term) ); |
331 |
|
|
post = (OpenIsisSet *)malloc( nterm * sizeof(*post) ); |
332 |
|
|
nterm = 0; |
333 |
|
|
nval = 0; |
334 |
|
|
while ( openIsisTerm( &x.r, db, pre ) && x.r.len ) { |
335 |
|
|
for ( i=0; i<x.r.len; i++, nterm++ ) { |
336 |
|
|
int cnt; |
337 |
|
|
OpenIsisSet *set = post+nterm; |
338 |
|
|
char *c = malloc( x.r.field[i].len + 1 ); |
339 |
|
|
|
340 |
|
|
memcpy( c, x.r.field[i].val, x.r.field[i].len ); |
341 |
|
|
c[ x.r.field[i].len ] = 0; |
342 |
|
|
term[ nterm ] = c; |
343 |
|
|
|
344 |
|
|
set->len = 0; |
345 |
|
|
openIsisQuery( set, db, c, OPENISIS_QRY_KEYEQ, 0 ); |
346 |
|
|
if ( 0 >= set->len ) { |
347 |
|
|
openIsisSMsg( 2, "no results for '%s'\n", c ); |
348 |
|
|
return 1; |
349 |
|
|
} |
350 |
|
|
for ( cnt = 0; cnt < set->len; cnt++ ) |
351 |
|
|
if ( nval < set->id[cnt] ) |
352 |
|
|
nval = set->id[cnt]; |
353 |
|
|
p += set->len; |
354 |
|
|
} |
355 |
|
|
} |
356 |
|
|
openIsisSMsg( fd, "%d postings max mfn %d\n", p, nval ); |
357 |
|
|
openIsisSMsg( fd, "%.3f sec\n", millis(&tv)/1000. ); |
358 |
|
|
|
359 |
|
|
val = (const char **)malloc( (1+nval) * sizeof(*val) ); |
360 |
|
|
for ( i=1; i<=nval; i++ ) { |
361 |
|
|
OpenIsisRec *r = openIsisReadRow( db, i ); |
362 |
|
|
val[i] = 0; |
363 |
|
|
if ( ! r ) { |
364 |
|
|
openIsisSMsg( 2, "no rec %d\n", i ); |
365 |
|
|
continue; |
366 |
|
|
} |
367 |
|
|
if ( 0 < r->len ) { |
368 |
|
|
char *c = malloc( r->field[0].len + 1 ); |
369 |
|
|
|
370 |
|
|
memcpy( c, r->field[0].val, r->field[0].len ); |
371 |
|
|
c[ r->field[0].len ] = 0; |
372 |
|
|
val[i] = c; |
373 |
|
|
} |
374 |
|
|
free( r ); |
375 |
|
|
} |
376 |
|
|
l = millis(&tv); |
377 |
|
|
openIsisSMsg( fd, "sequential read %d rows in %.3f seconds %d rows per sec\n", |
378 |
|
|
nval, l/1000., nval*1000/(l?l:1) ); |
379 |
|
|
|
380 |
|
|
def.start = 0; |
381 |
|
|
i = 0; |
382 |
|
|
run( &def ); |
383 |
|
|
l = millis(&tv); |
384 |
|
|
run( &def ); |
385 |
|
|
l = millis(&tv); |
386 |
|
|
run( &def ); |
387 |
|
|
l = millis(&tv); |
388 |
|
|
openIsisSMsg( fd, "in-thread run in %.3f seconds %d rows per sec\n", |
389 |
|
|
l/1000., p*1000/(l?l:1) ); |
390 |
|
|
#ifdef HAVE_THREADS |
391 |
|
|
#define passes 3 |
392 |
|
|
{ |
393 |
|
|
static int nThreads[] = { 8, 2, 1, 4 }; |
394 |
|
|
int res[ sizeof(nThreads)/sizeof(nThreads[0]) ]; |
395 |
|
|
pthread_t th[ 8 /* max. nThreads */ ]; |
396 |
|
|
threadarg arg[ sizeof(th)/sizeof(th[0]) ]; |
397 |
|
|
int j; |
398 |
|
|
|
399 |
|
|
|
400 |
|
|
for ( i=0; i<(int)(sizeof(nThreads)/sizeof(nThreads[0])); i++ ) |
401 |
|
|
res[i] = 0; |
402 |
|
|
for ( j=0; j<(int)(sizeof(th)/sizeof(th[0])); j++ ) |
403 |
|
|
#if 0 |
404 |
|
|
if ( ! (arg[j].ses = openIsisSesGet( -1, 0 )) ) { |
405 |
|
|
openIsisSMsg( fd, "could not get %dth session\n", j ); |
406 |
|
|
return 1; |
407 |
|
|
} |
408 |
|
|
#endif |
409 |
|
|
for ( pass=0; pass<passes; pass++ ) |
410 |
|
|
for ( i=0; i<(int)(sizeof(nThreads)/sizeof(nThreads[0])); i++ ) { |
411 |
|
|
int rps, avg; |
412 |
|
|
millis(&tv); |
413 |
|
|
assert( nThreads[i] <= (int)(sizeof(th)/sizeof(th[0])) ); |
414 |
|
|
for ( j=0; j<nThreads[i]; j++ ) { |
415 |
|
|
arg[j].start = j*nterm/nThreads[i]; |
416 |
|
|
if ( pthread_create( th+j, 0, run, arg+j ) ) |
417 |
|
|
th[j] = (pthread_t)0; |
418 |
|
|
} |
419 |
|
|
openIsisSMsg( fd, "started %d threads\n", nThreads[i] ); |
420 |
|
|
avg = 0; |
421 |
|
|
for ( j=0; j<nThreads[i]; j++ ) { |
422 |
|
|
int t; |
423 |
|
|
pthread_join( th[j], (void**)&t ); |
424 |
|
|
avg += t; |
425 |
|
|
} |
426 |
|
|
avg /= nThreads[i]; |
427 |
|
|
l = millis(&tv); |
428 |
|
|
rps = (int)(nThreads[i] * p * 1000 / avg); |
429 |
|
|
openIsisSMsg( fd, "joined %d threads avg %.3f after %.3f seconds %d rows per sec\n", |
430 |
|
|
nThreads[i], avg/1000., l/1000., rps ); |
431 |
|
|
res[i] += rps; |
432 |
|
|
} |
433 |
|
|
for ( i=0; i<(int)(sizeof(nThreads)/sizeof(nThreads[0])); i++ ) |
434 |
|
|
openIsisSMsg( fd, "%d threads %d rows per sec\n", nThreads[i], res[i]/passes ); |
435 |
|
|
} |
436 |
|
|
#else |
437 |
|
|
(void)pass; /* avoid compiler warning */ |
438 |
|
|
#endif /* HAVE_THREADS */ |
439 |
|
|
return 0; |
440 |
|
|
} /* crash */ |
441 |
|
|
|
442 |
|
|
|
443 |
|
|
/* ************************************************************ |
444 |
|
|
package functions |
445 |
|
|
*/ |
446 |
|
|
/* ************************************************************ |
447 |
|
|
public functions |
448 |
|
|
*/ |
449 |
|
|
int main ( int argc, const char **argv ) |
450 |
|
|
{ |
451 |
|
|
int ret = 0; |
452 |
|
|
int i,intarg=0; |
453 |
|
|
todo what = DO_DUMP; |
454 |
|
|
format fmt = FMT_TXT; |
455 |
|
|
int check = OPENISIS_CHK_FIX; |
456 |
|
|
int searchmode = OPENISIS_QRY_KEYAT; |
457 |
|
|
int idxmode = OPENISIS_IDXPF; |
458 |
|
|
const char *search = 0; |
459 |
|
|
const char *idxto = 0; |
460 |
|
|
int needdb = !0; |
461 |
|
|
const char *dowrite = 0; |
462 |
|
|
ifmode ifm = IFM_DUMP; |
463 |
|
|
OpenIsisRec *argr; |
464 |
|
|
OpenIsisDb *odb; |
465 |
|
|
|
466 |
|
|
/* initialize minimal env */ |
467 |
|
|
openIsisCOpen(0); |
468 |
|
|
argr = openIsisRSet( 0, |
469 |
|
|
OPENISIS_RFDT|OPENISIS_RARGV|OPENISIS_RIGN | (argc-1), |
470 |
|
|
openIsisFdtSyspar, argv+1 ); |
471 |
|
|
|
472 |
|
|
if ( 2 == argc && ! strcmp("-version",argv[1]) ) { |
473 |
|
|
openIsisSMsg( 0, "%s\n", OPENISIS_VERSION ); |
474 |
|
|
goto bye; |
475 |
|
|
} |
476 |
|
|
/* check options ... */ |
477 |
|
|
for ( i=0; i < argc; ) { |
478 |
|
|
const char *n = argv[i], *v = 0; |
479 |
|
|
assert( n ); |
480 |
|
|
if ( '-' == n[0] ) |
481 |
|
|
n++; |
482 |
|
|
if ( 1 == argc - i || '-' == argv[i+1][0] ) { /* no value */ |
483 |
|
|
i++; |
484 |
|
|
} else { |
485 |
|
|
v = argv[i+1]; |
486 |
|
|
assert( v ); |
487 |
|
|
i += 2; |
488 |
|
|
} |
489 |
|
|
|
490 |
|
|
if ( argchk("logfile",n,v) ) |
491 |
|
|
openIsisLog( '=', v ); |
492 |
|
|
else if ( argchk("v",n,v) ) |
493 |
|
|
openIsisLog( *v, 0 ); |
494 |
|
|
else if ( argchk("scan",n,v) ) { |
495 |
|
|
what = DO_SCAN; |
496 |
|
|
search = v; |
497 |
|
|
} |
498 |
|
|
else if ( argchk("search",n,v) ) { |
499 |
|
|
what = DO_SEARCH; |
500 |
|
|
search = v; |
501 |
|
|
} |
502 |
|
|
else if ( argchk("upto",n,v) ) { |
503 |
|
|
idxmode = OPENISIS_IDXUPTO; |
504 |
|
|
idxto = v; |
505 |
|
|
} |
506 |
|
|
else if ( argchk("incl",n,v) ) { |
507 |
|
|
idxmode = OPENISIS_IDXINCL; |
508 |
|
|
idxto = v; |
509 |
|
|
} |
510 |
|
|
else if ( argchk("query",n,v) ) { |
511 |
|
|
what = DO_SEARCH; |
512 |
|
|
searchmode = OPENISIS_QRY_SIMPLE; |
513 |
|
|
search = v; |
514 |
|
|
} |
515 |
|
|
else if ( argchk("terms",n,v) ) { |
516 |
|
|
what = DO_TERMS; |
517 |
|
|
search = v; |
518 |
|
|
} |
519 |
|
|
else if ( argchk("perf",n,v) ) { |
520 |
|
|
what = DO_PERF; |
521 |
|
|
intarg = atoi(v); |
522 |
|
|
} |
523 |
|
|
else if ( argchk("crash",n,v) ) { |
524 |
|
|
what = DO_CRASH; |
525 |
|
|
search = v; |
526 |
|
|
} |
527 |
|
|
else if ( argchk("split",n,v) ) { |
528 |
|
|
what = DO_SPLIT; |
529 |
|
|
search = v; |
530 |
|
|
needdb = 0; |
531 |
|
|
} |
532 |
|
|
else if ( argchk("fmt",n,v) ) { |
533 |
|
|
if ( ! strcmp("mfn",v) ) |
534 |
|
|
fmt =FMT_MFN; |
535 |
|
|
else if ( ! strcmp("mfnf",v) ) |
536 |
|
|
fmt =FMT_MFNF; |
537 |
|
|
else if ( ! strcmp("prop",v) ) |
538 |
|
|
fmt =FMT_PROP; |
539 |
|
|
else if ( ! strcmp("txt",v) ) |
540 |
|
|
fmt =FMT_TXT; |
541 |
|
|
else if ( ! strcmp("txtw",v) ) |
542 |
|
|
fmt =FMT_TXTW; |
543 |
|
|
} |
544 |
|
|
else if ( ! strcmp("check",n) ) |
545 |
|
|
what = DO_CHECK; |
546 |
|
|
else if ( ! strcmp("vutf",n) ) { |
547 |
|
|
what = DO_VUTF; |
548 |
|
|
needdb = 0; |
549 |
|
|
} else if ( argchk("pft",n,v) ) |
550 |
|
|
pft = v; |
551 |
|
|
else if ( ! strcmp("stream",n) ) { |
552 |
|
|
what = DO_STREAM; |
553 |
|
|
needdb = 0; |
554 |
|
|
} else if ( argchk("write",n,v) ) |
555 |
|
|
dowrite = v; |
556 |
|
|
else if ( argchk("append",n,v) ) { |
557 |
|
|
dowrite = v; |
558 |
|
|
append = !0; |
559 |
|
|
} else if ( ! strcmp("idxall",n) ) |
560 |
|
|
idxall = !0; |
561 |
|
|
else if ( ! strcmp("mfnlist",n) ) |
562 |
|
|
what = DO_MFNLIST; |
563 |
|
|
else if ( argchk("ifload",n,v) ) { |
564 |
|
|
what = DO_IFLOAD; |
565 |
|
|
intarg = atoi(v); |
566 |
|
|
} else if ( ! strcmp("swload",n) ) { |
567 |
|
|
what = DO_SWLOAD; |
568 |
|
|
} else if ( ! strcmp("ifadd",n) ) { |
569 |
|
|
what = DO_IFLOAD; |
570 |
|
|
intarg = -1; |
571 |
|
|
} else if ( ! strcmp("ifdel",n) ) { |
572 |
|
|
what = DO_IFLOAD; |
573 |
|
|
intarg = -2; |
574 |
|
|
} else if ( ! strcmp("ifcopy",n) ) { |
575 |
|
|
what = DO_IFDUMP; |
576 |
|
|
ifm = IFM_COPY; |
577 |
|
|
} else if ( ! strcmp("ifchk",n) ) { |
578 |
|
|
what = DO_IFDUMP; |
579 |
|
|
ifm = IFM_CHK; |
580 |
|
|
} else if ( ! strcmp("iftab",n) ) { |
581 |
|
|
what = DO_IFDUMP; |
582 |
|
|
ifm = IFM_TAB; |
583 |
|
|
} else if ( ! strcmp("ifdump",n) ) |
584 |
|
|
what = DO_IFDUMP; |
585 |
|
|
else if ( ! strcmp("noxi",n) ) |
586 |
|
|
ifm = IFM_OLD; |
587 |
|
|
else if ( argchk("out",n,v) ) { |
588 |
|
|
char buf[256] = ">"; |
589 |
|
|
int l = strlen(v); |
590 |
|
|
if ( l < 254 ) { |
591 |
|
|
memcpy( buf+1, v, l ); |
592 |
|
|
buf[l+2] = 0; |
593 |
|
|
openIsisSOpen( buf, 0, 0 ); |
594 |
|
|
} |
595 |
|
|
} |
596 |
|
|
else if ( argchk("fdtdump",n,v) ) { |
597 |
|
|
what = DO_FDT; |
598 |
|
|
intarg = atoi(v); |
599 |
|
|
} |
600 |
|
|
} /* while argc */ |
601 |
|
|
|
602 |
|
|
|
603 |
|
|
if ( needdb && 0 > (db = openIsisOpen( 0, argv + 1, argc - 1 )) ) { |
604 |
|
|
openIsisSMsg( 2, |
605 |
|
|
"openisis " OPENISIS_VERSION "\n\n" |
606 |
|
|
"please specify a valid database with -db, e.g.\n" |
607 |
|
|
"-db /winisis/data/cds\n" |
608 |
|
|
"\n" |
609 |
|
|
"other options are:\n" |
610 |
|
|
"-search term search for term\n" |
611 |
|
|
"-query \"query\" run a query like \"water * plant\"\n" |
612 |
|
|
"-terms term list terms matching term (e.g. plant$)\n" |
613 |
|
|
); |
614 |
|
|
/* warning: string length `580' is greater than the minimum length |
615 |
|
|
* `509' ISO C89 is required to support |
616 |
|
|
*/ |
617 |
|
|
openIsisSMsg( 2, |
618 |
|
|
"-fmt mfn for a search or query, list only the mfn\n" |
619 |
|
|
"-fmt mfnf for a search or query, list the mfn and 1st field\n" |
620 |
|
|
"-pft \"pft\" use printformat (currently very limited)\n" |
621 |
|
|
"-write dbpath specify a db where records are written to\n" |
622 |
|
|
"-mfnlist read mfns from stdin\n" |
623 |
|
|
"-ifload pctfree read .lk2-index from stdin\n" |
624 |
|
|
"\n" |
625 |
|
|
"default output format is one field per line like tag<TAB>value\n" |
626 |
|
|
); |
627 |
|
|
ret = 1; |
628 |
|
|
goto bye; |
629 |
|
|
} |
630 |
|
|
|
631 |
|
|
if ( dowrite && 0 > (wdb = openIsisOpen( dowrite, 0, 0 )) ) { |
632 |
|
|
openIsisSMsg( 2, "could not open write target db '%s'\n", dowrite ); |
633 |
|
|
ret = 2; |
634 |
|
|
goto bye; |
635 |
|
|
} |
636 |
|
|
|
637 |
|
|
|
638 |
|
|
switch ( what ) { |
639 |
|
|
case DO_DUMP: { |
640 |
|
|
int max = openIsisMaxRowid( db ); |
641 |
|
|
int rowid; |
642 |
|
|
for ( rowid = 1; rowid <= max; rowid++ ) |
643 |
|
|
printid( rowid, fmt ); |
644 |
|
|
} break; /* DO_DUMP */ |
645 |
|
|
case DO_MFNLIST: { |
646 |
|
|
char *buf = 0; |
647 |
|
|
int l; |
648 |
|
|
while ( 0 <= (l = openIsisSReadln( &buf )) ) { |
649 |
|
|
int id = 0; |
650 |
|
|
while ( l-- ) |
651 |
|
|
id = 10*id + *buf++ - '0'; |
652 |
|
|
if ( id ) |
653 |
|
|
printid( id, fmt ); |
654 |
|
|
} |
655 |
|
|
} break; /* DO_MFNLIST */ |
656 |
|
|
case DO_SCAN: { |
657 |
|
|
int max = openIsisMaxRowid( db ); |
658 |
|
|
int rowid; |
659 |
|
|
for ( rowid = 1; 0 < rowid && rowid <= max; rowid++ ) |
660 |
|
|
rowid = print( openIsisScan( db, rowid, 0, search ), !0, fmt ); |
661 |
|
|
} break; /* DO_SCAN */ |
662 |
|
|
case DO_SEARCH: { |
663 |
|
|
int cnt; |
664 |
|
|
OpenIsisSet set; |
665 |
|
|
set.len = 0; |
666 |
|
|
openIsisQuery( &set, db, search, searchmode, 0 ); |
667 |
|
|
if ( 0 >= set.len ) { |
668 |
|
|
openIsisSMsg( 2, "no results for '%s'\n", search ); |
669 |
|
|
ret = 1; |
670 |
|
|
goto bye; |
671 |
|
|
} |
672 |
|
|
/* openIsisSMsg( 2, "%d\trows for\t%s\n", set.len, search ); */ |
673 |
|
|
for ( cnt = 0; cnt < set.len; cnt++ ) |
674 |
|
|
printid( set.id[cnt], fmt ); |
675 |
|
|
} break; /* DO_SEARCH */ |
676 |
|
|
case DO_TERMS: { |
677 |
|
|
union { OpenIsisRec r; char buf[10000]; } x; |
678 |
|
|
x.r.len = 0; |
679 |
|
|
x.r.bytes = sizeof(x); |
680 |
|
|
while ( openIsisTerm( &x.r, db, search ) && x.r.len ) { |
681 |
|
|
/* openIsisSMsg( 1, "%d terms\n", x.r.len ); */ |
682 |
|
|
for ( i=0; i<x.r.len; i++ ) |
683 |
|
|
openIsisSMsg( 1, "%.*s\n", (int)x.r.field[i].len, x.r.field[i].val ); |
684 |
|
|
} |
685 |
|
|
} break; /* DO_TERMS */ |
686 |
|
|
case DO_PERF: { |
687 |
|
|
int max = openIsisMaxRowid( db ); |
688 |
|
|
while ( 0 < intarg-- ) { |
689 |
|
|
OpenIsisRec *r = openIsisReadRow( db, 1+((int)rand() % max) ); |
690 |
|
|
free( r ); |
691 |
|
|
} |
692 |
|
|
} break; /* DO_PERF */ |
693 |
|
|
case DO_CHECK: |
694 |
|
|
ret = openIsisCheck( db, check ); |
695 |
|
|
goto bye; |
696 |
|
|
case DO_CRASH: |
697 |
|
|
ret = crash( search ); |
698 |
|
|
goto bye; |
699 |
|
|
case DO_SPLIT: { |
700 |
|
|
OpenIsisField f; |
701 |
|
|
OpenIsisRec *r; |
702 |
|
|
f.tag = 24; f.val = search; f.len = strlen(search); |
703 |
|
|
r = openIsisReadField( 0, &f ); |
704 |
|
|
if ( r ) |
705 |
|
|
for ( i=0; i<r->len; i++ ) |
706 |
|
|
openIsisSMsg( 1, "%c=%.*s\n", (int)r->field[i].tag, |
707 |
|
|
(int)r->field[i].len, r->field[i].val ); |
708 |
|
|
} break; |
709 |
|
|
case DO_STREAM: { |
710 |
|
|
OpenIsisIos ios; |
711 |
|
|
OpenIsisRecStream rs = { 0, OPENISIS_STOPONEMPTY, 0, 0, 0 }; |
712 |
|
|
LIO_SINIT( &ios, lio_stdio, "stdin", LIO_IN ); |
713 |
|
|
rs.in = &ios; /* some gcc versions need it this way */ |
714 |
|
|
while ( 0 < (i = openIsisSGetr( &rs )) ) |
715 |
|
|
print( rs.rec, 0, fmt ); |
716 |
|
|
} break; |
717 |
|
|
case DO_IFLOAD: |
718 |
|
|
case DO_SWLOAD: { |
719 |
|
|
OpenIsisKey key; |
720 |
|
|
OpenIsisHit hit; |
721 |
|
|
OpenIsisIndex idx = openIsisIdxOpen( db, intarg ); |
722 |
|
|
char *buf = 0; |
723 |
|
|
int l, lines = 0; |
724 |
|
|
|
725 |
|
|
memset( &hit, 0, sizeof(hit) ); |
726 |
|
|
hit.dbn = (-2 == intarg) ? 0xffff : 0; /* secret key for ifdel */ |
727 |
|
|
while ( 0 <= (l = openIsisSReadln( &buf )) && buf ) { |
728 |
|
|
char *t = memchr( buf, '\t', l ); |
729 |
|
|
if ( DO_SWLOAD == what ) { |
730 |
|
|
memcpy( key.byt, buf, key.len = (unsigned char)l ); |
731 |
|
|
} else if ( t ) { /* tab delimited */ |
732 |
|
|
key.len = (unsigned char)(t - buf); |
733 |
|
|
memcpy( key.byt, buf, key.len ); |
734 |
|
|
if ( 0 >= (l -= t-buf+1) || !(t = memchr( buf=t+1, '\t', l )) ) |
735 |
|
|
continue; |
736 |
|
|
hit.mfn = (unsigned) openIsisA2i( buf, t-buf ); |
737 |
|
|
if ( 0 >= (l -= t-buf+1) || !(t = memchr( buf=t+1, '\t', l )) ) |
738 |
|
|
continue; |
739 |
|
|
hit.tag = (unsigned short) openIsisA2i( buf, t-buf ); |
740 |
|
|
if ( 0 >= (l -= t-buf+1) || !(t = memchr( buf=t+1, '\t', l )) ) |
741 |
|
|
continue; |
742 |
|
|
hit.occ = (unsigned short) openIsisA2i( buf, t-buf ); |
743 |
|
|
if ( 0 >= (l -= t-buf+1) ) |
744 |
|
|
continue; |
745 |
|
|
hit.pos = (unsigned short) openIsisA2i( t+1, l ); |
746 |
|
|
} else { |
747 |
|
|
/* 10/30 key BLANK 7 mfn BLANK 5 tag BLANK 4 occ BLANK 4 pos*/ |
748 |
|
|
int eok = l - 24; /* pos of blank after key, 10 or 30 */ |
749 |
|
|
if ( 54 != l && 34 != l ) { |
750 |
|
|
openIsisSMsg( OPENISIS_ERR_INVAL, |
751 |
|
|
"bad ifload input len %d, want 34 or54 bytes + newline\n", l ); |
752 |
|
|
break; |
753 |
|
|
} |
754 |
|
|
for ( i=eok-1; ' ' == buf[i] && i--; ) |
755 |
|
|
; |
756 |
|
|
key.len = (unsigned char) (++i); |
757 |
|
|
memcpy( key.byt, buf, key.len ); |
758 |
|
|
log_msg( LOG_VERBOSE, "'%.*s'", 7, buf+eok+1 ); |
759 |
|
|
hit.mfn = (unsigned) openIsisA2i( buf+eok+1, 7 ); |
760 |
|
|
hit.tag = (unsigned short)openIsisA2i( buf+eok+9, 5 ); |
761 |
|
|
hit.occ = (unsigned short)openIsisA2i( buf+eok+15, 4 ); |
762 |
|
|
hit.pos = (unsigned short)openIsisA2i( buf+eok+20, 4 ); |
763 |
|
|
} |
764 |
|
|
log_msg( LOG_VERBOSE, "'%.*s' %d %d %d %d", |
765 |
|
|
key.len, key.byt, hit.mfn, hit.tag, hit.occ, hit.pos ); |
766 |
|
|
if ( openIsisIdxAdd( idx, &key, &hit ) ) |
767 |
|
|
break; |
768 |
|
|
if ( !(++lines & 0x3ff) ) |
769 |
|
|
log_msg( LOG_INFO, "%dK lines", lines >> 10 ); |
770 |
|
|
} |
771 |
|
|
openIsisIdxDone( idx ); |
772 |
|
|
} break; |
773 |
|
|
case DO_IFDUMP: { |
774 |
|
|
OpenIsisIdxLoop l; |
775 |
|
|
memset( &l, 0, sizeof(l) ); |
776 |
|
|
l.flg = idxmode; |
777 |
|
|
switch ( ifm ) { |
778 |
|
|
case IFM_OLD: |
779 |
|
|
l.flg |= OPENISIS_IDXTRAD; |
780 |
|
|
case IFM_DUMP: |
781 |
|
|
l.cb = (OpenIsisIdxCb*)printlk2; |
782 |
|
|
break; |
783 |
|
|
case IFM_TAB: |
784 |
|
|
l.cb = (OpenIsisIdxCb*)printtab; |
785 |
|
|
break; |
786 |
|
|
case IFM_COPY: |
787 |
|
|
l.flg |= OPENISIS_IDXTRAD; |
788 |
|
|
l.me = openIsisIdxOpen( 0 <= wdb ? wdb : db, 0 ); |
789 |
|
|
l.cb = (OpenIsisIdxCb*)openIsisIdxAdd; |
790 |
|
|
break; |
791 |
|
|
case IFM_CHK: |
792 |
|
|
/* nuttin */ |
793 |
|
|
break; |
794 |
|
|
} |
795 |
|
|
if ( search ) |
796 |
|
|
memcpy( l.key.byt, search, |
797 |
|
|
l.key.len = (unsigned char)strlen( search ) ); |
798 |
|
|
if ( idxto ) |
799 |
|
|
memcpy( l.to.byt, idxto, |
800 |
|
|
l.to.len = (unsigned char)strlen( idxto ) ); |
801 |
|
|
openIsisIdxLoop( db, &l ); |
802 |
|
|
if ( IFM_COPY == ifm ) |
803 |
|
|
openIsisIdxDone( (OpenIsisIndex)l.me ); |
804 |
|
|
} break; |
805 |
|
|
case DO_FDT: |
806 |
|
|
odb = ldb_getdb( db ); |
807 |
|
|
if ( odb && odb->fdt ) |
808 |
|
|
print( openIsisFFdt2Rec( odb->fdt, 0, intarg ), 0, fmt ); |
809 |
|
|
break; |
810 |
|
|
case DO_VUTF: { |
811 |
|
|
char buf[1024]; |
812 |
|
|
int t = 0, f = 0, g; |
813 |
|
|
while ( 0 < (g = lio_read( &lio_in, buf, sizeof(buf) )) ) { |
814 |
|
|
int l = openIsisValidUTF8( buf, g, &f ); |
815 |
|
|
if ( l ) { |
816 |
|
|
openIsisSMsg( OPENISIS_ERR_INVAL, |
817 |
|
|
"at total %d = %d+%d\n", l-1+t, l-1, t ); |
818 |
|
|
ret = 1; |
819 |
|
|
goto bye; |
820 |
|
|
} |
821 |
|
|
t += g; |
822 |
|
|
} |
823 |
|
|
} |
824 |
|
|
} /* switch ( what ) */ |
825 |
|
|
bye: |
826 |
|
|
if ( 0 <= db ) |
827 |
|
|
openIsisClose( db ); |
828 |
|
|
if ( 0 <= wdb ) |
829 |
|
|
openIsisClose( wdb ); |
830 |
|
|
|
831 |
|
|
/* at least with WINE, |
832 |
|
|
atexit cleanup is not performed |
833 |
|
|
unless we explicitly call exit :( |
834 |
|
|
*/ |
835 |
|
|
exit( ret ); |
836 |
|
|
return ret; |
837 |
|
|
} /* openisis main */ |