Revision 238 (by dpavlin, 2004/03/08 17:46:16) tagging openisis 0.9.0
/*
	openisis - an open implementation of the CDS/ISIS database
	Version 0.8.x (patchlevel see file Version)
	Copyright (C) 2001-2003 by Erik Grziwotz, erik@openisis.org

	This library is free software; you can redistribute it and/or
	modify it under the terms of the GNU Lesser General Public
	License as published by the Free Software Foundation; either
	version 2.1 of the License, or (at your option) any later version.

	This library is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
	Lesser General Public License for more details.

	You should have received a copy of the GNU Lesser General Public
	License along with this library; if not, write to the Free Software
	Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

	see README for more information
EOH */

/*
	$Id: lrec.c,v 1.53 2003/05/29 18:03:35 kripke Exp $
	implementation of record cooking.
*/

#include <stddef.h>
#include <stdarg.h>
#include <stdio.h> /* vsnprintf */
#include <stdlib.h> /* free */
#include <string.h> /* memset et al */

#ifdef WIN32
#	define vsnprintf _vsnprintf
#endif /* WIN32 */

#include "ldb.h"
#include "lio.h"
#include "lcs.h"


/* ************************************************************
	private types
*/

typedef struct {
	int len;      /* length of tagf */
	int tagf[2];  /* tag and modification flags */
} LrecMF;

#define MF_SET    0x01
#define MF_DEL    0x02

/* ************************************************************
	private data
*/
/* ************************************************************
	private functions
*/

static LrecMF* mfCtor (Rec *rec) {
	int     len = (int)(2 * rec->len);
	int    *M;
	Field  *F;
	LrecMF *lmf = (LrecMF*) mAlloc ((1 + len) * sizeof (int));
	if (0 == lmf) {
		return 0;
	}
	memset (lmf->tagf, 0, len * sizeof (int));
	lmf->len = len;
	for (M = lmf->tagf, F = rec->field; len > M - lmf->tagf; ++F, M += 2) {
		*M = (int) F->tag;
	}
	return lmf;
}

static int rCompact (Rec *rec) {
	int rt = 0;
	if (0 != rec) {
		char  *V    = (char*)rec + rec->base;
		Field *F    = rec->field;
		Field *L    = F + rec->len;
		int    used = rec->base;
		while (F < L) {
			if (V != F->val) {
				rt = !0;
				break;
			}
			V += F->len;
			used += F->len;
			++F;
		}
		if (rt) {
			char *buf = (char*) mAlloc (rec->used - (V - (char*)rec));
			char *B   = buf;
			int   nb  = 0;
			if (0 == buf) {
				return 0;
			}
			while (F < L) {
				memcpy (B, F->val, F->len);
				F->val = V;
				B += F->len;
				V += F->len;
				nb += F->len;
				++F;
			}
			memcpy ((char*)rec + used, buf, nb);
			rec->used = used + nb;
			mFree (buf);
		}
	}
	return rt;
}

/**	@return
		-2 - malloc failed,
		-1 - tag not present,
		0 - compactification not neccessary,
		1 - rec data buffer has gaps
*/
static int rReplace (
	Rec **rec, LrecMF *lmf,
	int tag, int occr, int lmode, const char *val, int disc
) {
	Rec   *nr;
	Field *F;
	int   *M;
	int    vlen;
	int    fnd  = -1;
	int    ocnt = -1;
	for (M = lmf->tagf; lmf->len > M - lmf->tagf; M += 2) {
		if (tag == M[0]) {
			if (0 <= occr) {
				if (++ocnt == occr) {
					M[1] = lmode;
					fnd = (M - lmf->tagf) / 2;
					break;
				}
			}
			else {
				if (0 == (MF_SET & M[1])) {
					M[1] = lmode;
					fnd = (M - lmf->tagf) / 2;
					if (lmode) {
						for (M += 2; lmf->len > M - lmf->tagf; M += 2) {
							if (tag == M[0]) {
								M[1] = MF_DEL;
							}
						}
					}
					break;
				}
			}
		}
	}
	if (0 > fnd) {
		return -1;
	}
	if (MF_SET != lmode) {
		return 1;
	}
	F = (*rec)->field + fnd;
	vlen = strlen (val);
	if (F->len == vlen) {
		memcpy ((char*)F->val, val, vlen);
		return 0;
	}
	if (F->len > vlen) {
		memcpy ((char*)F->val, val, vlen);
		F->len = vlen;
		return 1;
	}
	if ((*rec)->bytes < (*rec)->used + vlen) {
		nr = rDup (*rec, vlen, disc);
		if (0 == nr) {
			return -2;
		}
		F = (*rec = nr)->field + fnd;
	}
	F->val = (char*)(*rec) + (*rec)->used;
	F->len = vlen;
	memcpy ((char*)F->val, val, vlen);
	(*rec)->used += vlen;
	return 1;
}

static int rDel (Rec *rec, LrecMF *lmf) {
	Field  *F;
	int    *M;
	int     idx;
	int     dcnt = 0;
	int     done = 0;
	for (M = lmf->tagf; lmf->len > M - lmf->tagf; M += 2) {
		if (0 != (MF_DEL & M[1])) {
			idx = (M - lmf->tagf) / 2 - dcnt;
			if (idx < rec->len - 1) {
				F = rec->field + idx;
				memmove (F, F + 1, (rec->len - idx - 1) * sizeof (Field));
			}
			--(rec->len);
			++dcnt;
			done = !0;
		}
	}
	return done;
}

/* ************************************************************
	package functions
*/

/* ************************************************************
	public functions
*/


Field *rGet ( Rec *r, int tag, int *pos )
{
	if (r) {
		int p = pos ? *pos : 0;
		for ( ; p < r->len; p++ )
			if ( tag == r->field[p].tag ) {
				if ( pos )
					*pos = p+1;
				return &r->field[p];
			}
		if ( pos )
			*pos = r->len;
	}
	return 0;
}	/* rGet */

Field *rOccurence ( Rec *r, int tag, int occ ) {
	Field *F;
	int    pos = 0;
	do {
		F = rGet (r, tag, &pos);
	} while (F && 0 <= --occ);
	return F;
}

char *rString (Rec *rec, int tag, int *pos, char *buf, int len) {
	Field *F = rGet (rec, tag, pos);
	if (!F) {
		return 0;
	}
	if (len > 1 + F->len) {
		len = 1 + (int) F->len;
	}
	if (0 >= --len) {
		*buf = 0;
		return buf;
	}
	strncpy (buf, F->val, len) [len] = 0;
	return buf;
}


char *rString2 (Rec *rec, Rec *dflt, int tag, char *buf, int len) {
	char *rt = rString (rec, tag, 0, buf, len);
	if (rt) {
		return rt;
	}
	return rString (dflt, tag, 0, buf, len);
}


int rInt ( Rec *r, int tag, int def, int *pos )
{
	if (r) {
		int p = pos ? *pos : 0;
		int rt;
		for ( ; p < r->len; p++ )
			if ( tag == r->field[p].tag ) {
				if ( pos )
					*pos = p+1;
				rt = a2i( r->field[p].val, r->field[p].len );
				if (rt) {
					return rt;
				}
				if (0 < luti_true (r->field[p].val, r->field[p].len)) {
					return 1;
				}
				return 0;
			}
		if ( pos )
			*pos = r->len;
	}
	return def;
}	/* rInt */


int rInt2 (Rec *rec, Rec *dflt, int tag, int def) {
	int rt1 = rInt (rec, tag, def, 0);
	if (def == rt1) {
		return rInt (dflt, tag, def, 0);
	}
	return rt1;
}


int rEnum (Fdt *fdt, Rec *rec, int tag, int def, int *pos) {
	char    buf[OPENISIS_FD_NAMELEN];
	Field  *F;
	Fd     *D;
	int     len, ev;
	if (! fdt || ! rec) {
		return def;
	}
	D = fById (fdt, tag, 0);
	if (! D) {
		return def;
	}
	F = rGet (rec, tag, pos);
	if (! F) {
		return def;
	}
	len = (int) F->len;
	if (OPENISIS_FD_NAMELEN <= len) {
		len = OPENISIS_FD_NAMELEN - 1;
	}
	strncpy (buf, F->val, len) [len] = 0;
	ev = fEnum (fdt, tag, buf);
	if (NOENUM == ev) {
		return def;
	}
	return ev;
}


Rec* rDup ( Rec *r, int room, int discard )
{
	Rec *nr = 0;
	int nfields, nbytes, hadfields, hadcontent;
	if ( ! r ) {
		nfields = 80; /* for a 1K base */
		nbytes = 8*1024;
		if ( nbytes < BASESZ(80) + room*3/2 )
			nbytes = BASESZ(80) + room*3/2;
		hadfields = 0;
		hadcontent = 0;
	} else {
		rCompact (r);
		hadfields = r->len;
		hadcontent = r->used - r->base;
		if ( 0 > room ) { /* shrink to fit */
			nfields = r->len;
			nbytes = BASESZ(nfields) + hadcontent;
		} else {
			nfields = 54 > r->len ? 80 : (r->len * 3 / 2); /* add 50% */
			if ( nfields < r->fields )
				nfields = r->fields;
			nbytes = 6*1024 > r->bytes ? 8*1024 : (r->bytes *3 / 2);
			if ( nbytes < r->used + room )
				nbytes = r->used + room*3/2;
			nbytes += (nfields - r->fields)*sizeof(Field);
		}
	}
	LOG_DBG( LOG_DEBUG,
		"extending rec size %d (cont %d) to %d bytes, %d -> %d fields",
		!r ? -1 : r->bytes, hadcontent, nbytes, hadfields, nfields );
	assert( nbytes >= BASESZ( nfields ) + hadcontent + room );
	if ( ! (nr = mAlloc( nbytes )) )
		return 0;
	memset( nr, 0, nbytes ); /* paranoia */
	nr->bytes = nbytes;
	nr->fields = nfields;
	nr->base = BASESZ( nfields );
	nr->used = nr->base + hadcontent;
	nr->len = hadfields;
	if ( ! r ) {
		nr->dbid = -1; /* no valid dbid if new rec */
	}
	else {
		char *obuf = ((char*)r) + r->base;
		char *nbuf = ((char*)nr) + nr->base;
		nr->dbid = r->dbid;
		/* for ! r, the following are 0 by memset */
		nr->rowid = r->rowid;
		nr->state = r->state;
		if ( hadcontent )
			memcpy( nbuf, obuf, hadcontent );
		if ( hadfields ) {
			int i;
			int displace = nbuf - obuf; /* ptrdiff_t */
			Field *f = nr->field;
			memcpy( nr->field, r->field, hadfields*sizeof(Field) );
			for ( i=hadfields; i--; )
				if ( f[i].val )
					f[i].val += displace;
#ifndef NDEBUG
			{
				char *end = ((char*)nr) + nr->used;
				for ( i=hadfields; i--; )
					if ( f[i].val
						&& (f[i].val < nbuf || f[i].val + f[i].len > end)
					) {
						int wasok = r->field[i].val >= obuf
						&& r->field[i].val + r->field[i].len <= obuf + hadcontent;
						sMsg( LOG_ERROR, "OOPS! nuked field %d which previously was %s",
							i, wasok ? "ok" : "already broken" );
						return 0; /* no cleanup, we're nearly dead anyway */
					}
			}
#endif
		}
		if ( discard )
			free( r );
	}
	assert( RECOK(nr) );
	return nr;
}	/* rDup */


Rec* rMsg ( Rec *r, int discard, int tag, const char *fmt, ... )
{
	char buf[1024];
	int l;
	va_list ap;
	va_start( ap, fmt );
	l = vsnprintf( buf, sizeof(buf), fmt, ap );
	if ( 0 > l ) /* older versions return -1 */
		l = sizeof(buf);
	va_end( ap );
	RADD( r, tag, buf, l, discard );
	return r;
}	/* rMsg */

/*
#define openIsisPrintf( r, d, t, f, a... ) \
	openIsisRMsg( OPENISIS_SES0(), r, d, t, f, ## a )
	requires gcc ...
*/
Rec* openIsisPrintf ( Rec *r, int discard, int tag, const char *fmt, ... )
{
	char buf[1024];
	int l;
	va_list ap;
	va_start( ap, fmt );
	l = vsnprintf( buf, sizeof(buf), fmt, ap );
	if ( 0 > l ) /* older versions return -1 */
		l = sizeof(buf);
	va_end( ap );
	RADD( r, tag, buf, l, discard );
	return r;
}	/* openIsisPrintf */


Rec *rAddI (Rec *rec, int tag, int val, int discard) {
	char  buf[32];
	int   len = i2a (buf, (int)val);
	RADD (rec, tag, buf, len, discard);
	return rec;
}


Rec* rSet ( Rec *oldr, int mode, ... )
{
	const char **argv = 0;
	LrecMF *lmf       = 0;
	Fdt    *fdt       = 0;
	Rec    *newr      = oldr;
	int     argc      = 0xffff & mode;
	int     dis       = RDIS & mode;
	int     op        = ROP & mode;
	int     lmode     = MF_SET;
	int     compact   = 0;

	va_list ap;
	va_start( ap, mode );

	if (RFDT & mode) {
		fdt = va_arg( ap, Fdt* );
	}
	if (RARGV & mode) {
		argv = va_arg( ap, const char** );
	}
	if (! fdt) {
		fdt = luti_fdt_from_rec (oldr);
	}
	if (op) {
		switch (op) {
		case RDEL:
			lmode = MF_DEL;
			break;
		case RCHG:
			break;
		case RDFLT:
			lmode = 0;
			break;
		default:
			sMsg (ERR_IDIOT, "rSet: illegal mode %x", op);
			return newr;
		}
		if (oldr) {
			if (0 == (lmf = mfCtor (oldr))) {
				goto done;
			}
		}
		else if (RDEL == op) {
			goto done;
		}
	}

	for ( ;; ) {
		char   ib[16];
		Rec   *tmpr;
		const char *v = 0;
		int    id   = -1;
		int    occr = -1;
		Fd    *fd   = 0;
		int    rpl;
		if ( argv ) {
			if ( 1 > argc )
				break;
			v = luti_parse_path (*argv, fdt, &fd, &id, &occr);
			if (!v || *v) {
				fd = 0;
				id = -1;
			}
			v = 0;
			++argv;
			--argc;
		}
		else {
			id = va_arg( ap, int );
			if ( ! id )
				break;
			fd = fdt ? fById( fdt, id, 0 ) : 0;
			if (RDEL != op) {
				v = va_arg( ap, char* );
			}
		}
		if ( fd ) {
			if (RDEL != op) {
				int e;
				if (argv) {
					if ( 1 > argc ) {
						if (FTB != fd->type) {
							break;
						}
					}
					else {
						v = *argv++;
						--argc;
					}
				}
				switch ( fd->type ) {
				case FTE:
					e = fEnum( fdt, fd->id, v );
					if ( NOENUM == e ) {
						if ( RIGN & mode )
							continue;
						sMsg( LOG_ERROR,
							"bad enum value '%s' for id %d", v, fd->id );
						goto error;
					}
					i2a( ib, e );
					v = ib;
					break;
				case FTB:
					e = v ? luti_true (v, -1) : 1;
					if (0 > e) {
						/* non-given values default to true */
						++argc;
						--argv;
						v = "1";
					}
					else {
						v = e ? "1" : "0";
					}
					break;
				}
			} /* RDEL != op */
		} /* fd */
		else {
			if (argv) {
				if (RDEL != op) {
					if ( 1 > argc ) {
						break;
					}
					v = *argv++;
					--argc;
				}
			}
			if (fdt /*|| 0 >= id*/) {
				if ( RIGN & mode )
					continue;
				if ( argv )
					sMsg( LOG_ERROR, "unknown field name '%s'",
						argv[RDEL == op ? -1 : -2] );
				else
					sMsg( LOG_ERROR, "unknown field id %d", id );
				break;
			}
		}
		if (!v) {
			v = "";
		}
		if (op) {
			rpl = lmf ? rReplace (
				&newr, lmf, id, occr, lmode, v, dis || newr != oldr) :
				-1;
			if (RDFLT == op) {
				if (-1 != rpl) {
					continue;
				}
			}
			else {
				if (-2 == rpl) {
					goto done;
				}
				if (-1 != rpl) {
					if (1 == rpl) {
						compact = !0;
					}
					continue;
				}
				if (RDEL == op) {
					continue;
				}
			}
			/* fall thru */
		}
		tmpr = newr;
		RADDS( newr, id, v, dis || newr != oldr );
		if (0 == newr) {
			goto done;
		}
		if (newr != tmpr) {
			compact = 0;
		}
		sMsg( LOG_VERBOSE,
			"added v '%s' id %d as %dth", v, id, newr->len );
	}

error:
	if (lmf) {
		if (rDel (newr, lmf)) {
			compact = !0;
		}
	}

	if (compact && 0 == (RNOC & mode)) {
		rCompact (newr);
	}

done:
	if (lmf) {
		mFree (lmf);
	}
	va_end( ap );
	return newr;
}	/* rSet */


Rec *dFmt ( Rec *buf, const char *fmt, int db, int rowid )
{
	Rec *r = dRead( db, rowid );
	Rec *q;
	if ( ! r )
		return 0;
	q = rFmt( buf, fmt, r );
	free( r );
	return q;
}	/* dFmt */


Rec *dScan ( int db, int rowid, int tag, const char *txt )
{
	int max = dMaxId( db );
	int tlen = strlen( txt );
	const char f = *txt;

	for ( ; rowid <= max; rowid++ ) {
		int i, found = 0;
		Rec *r = dRead( db, rowid );
		if ( ! r )
			continue;

		for ( i=0; i < r->len; i++ ) {
			const char *c, *e;
			if ( found
				|| (tag > 0 && tag != r->field[i].tag)
				|| tlen > r->field[i].len
			)
				continue;
			c = r->field[i].val;
			e = c + r->field[i].len - tlen;
			for ( ; c<=e; c++ )
				if ( f == *c && !memcmp( c, txt, tlen ) ) {
					found = !0;
					break;
				}
		}
		if ( found )
			return r;
	}
	return 0;
}	/* dScan */





Rec *rSplitf ( Rec *r, const Field* field )
{
	int i;
	int nfields = 0; /* number of subfields */
	int size; /* byte length of field list */
	const char *p, *e;

	if ( ! field || 2 > field->len )
		return 0;
	if ( '^' != field->val[0] ) /* initial anonymous subfield */
		nfields = 1;
	/* go counting hats ... */
	for ( e = (p=field->val) + field->len - 1; p<e; )
		if ( '^' == *p++ && '^' != *p ) /* ignore first of ^^ */
			nfields++;
	if ( '^' == *p )
		sMsg( LOG_ERROR, "found trailing '^' in field '%.*s'",
			(int)field->len, field->val );

	if ( r ) {
		if ( 1 > r->len )
			return 0;
		size = sizeof(Rec) + (r->len - 1) * sizeof(Field);
		if ( nfields > r->len )
			nfields = r->len;
	} else {
		/* first field already counted in sizeof(Rec) */
		size = sizeof(Rec) + (nfields - 1) * sizeof(Field);
		r = (Rec *)mAlloc( size );
		if ( ! r )
			return 0;
	}
	memset( r, 0, size );
	r->rowid = 0;
	r->len   = nfields;

	e = (p = field->val) + field->len;
	if ( '^' == *p ) /* else initial anonymous subfield */
		p++;
	for ( i=0; i<nfields; i++ ) {
		/* p is on hat or end ... skip all consecutive hats */
		while ( p<e && '^' == *p )
			p++;
		if ( p >= e ) {
			sMsg( LOG_ERROR,
				"confused cats hound counting hats at %d in '%.*s'",
				i, field->len, field->val );
			r->len = i;
			break;
		}
		/* p is after a hat */
		r->field[ i ].tag = p==field->val ? 0 : (unsigned char)*p++;
		r->field[ i ].val = p;
		/* advance in buf */
		while ( p<e && '^' != *p )
			p++;
		r->field[ i ].len = p - r->field[ i ].val;
		/* log_msg( LOG_ERROR, "subf %d of %d tag %c off %d",
			i, nfields, r->field[ i ].tag, r->field[ i ].val - field->val ); */
	}

	return r;
}	/* rSplitf */




enum { /* stream reading state */
	RS_REC, /* not in any record */
	RS_TAG, /* in tag: done is at beginning of line */
	RS_SEP, /* in sep: done is at beginning of sep */
	RS_VAL, /* done is somewhere in the value ... */
	RS_EOL  /* not really a state, but a flag */
};


/**
	read a record from a stream.
	This may either
	<ol>
	<li> read a record and return 1
	</li>
	<li> not get enough bytes on a non-blocking stream and return 0
	</li>
	<li> have some error and return negative
	</li>
	</ol>
*/
int sGetr ( OpenIsisRecStream *s )
{
	Ios *lio = s->in;
	int state = 3 & s->flg;
	int emptyline = 0;
	int ret = 0;
	if ( ! s->rec
		&& ! (s->rec = s->buf) /* have buffer ? */
		&& ! (s->rec = rDup( 0, 0, 0 ))
		/* allocate and prepare an 8k standard record */
	) {
		ret = sMsg( LOG_ERROR, "could not get record" );
		goto fatal;
	}
	if ( RS_REC == state ) { /* initialize */
		s->rec->rowid = 0;
		s->rec->len = 0; /* nuke all fields */
		s->rec->used = s->rec->base; /* nuke value buffer */
		/* let base/fields untouched, it hopefully has a reasonable value */
		state = RS_TAG;
	}
	if ( ! s->rec->len )
		state = RS_TAG;
	for (;;) {
		unsigned char *b, *p, *e; /* begin, pointer, end */
		if ( lio->b.done >= lio->b.fill ) {
	reload:
			if ( ! LIO_FILL( lio ) )
				goto done; /* EAGAIN -- see ya later */
			/* so we have bytes or EOF */
			if ( lio->b.done >= lio->b.fill ) {
				if ( LIO_IN & lio->file ) {
					sMsg( LOG_ERROR, "confused in instream: no bytes and no EOF" );
					lio_close( &lio->file, LIO_IN );
				}
				break;
			}
		}
		/* so we have bytes */
		p = b = lio->b.c + lio->b.done;
		e = lio->b.c + lio->b.fill;
		/* try to eat one line */
		switch ( state ) {
		case RS_TAG: {
			int tag = -1;
			int recognized = 0;
			if ( LCS_ISFR(lcs_latin1_ct[*p]) ) { /* line starts with line/record sep */
				if ( '\n' == *p && (RS_EOL & s->flg) ) { /* TODO */
					s->flg &= ~RS_EOL;
					continue;
				}
				emptyline = !0;
				break;
			}
			for (;;p++) {
				if ( p < e ) {
					if ( LCS_ISWORD(lcs_latin1_ct[*p]) )
						continue; /* tight loop */
					break;
				}
				if ( 4096 > p - b )
					goto reload;
				sMsg( ERR_TRASH, "tag too long" );
				goto fatal;
			}
			/* identify tag between s and p */
			if ( p == b )
				recognized = !0;/* empty tag */
			else if ( '0' <= *b && *b <= '9' ) {
				unsigned char *d = b; /* digit ? */
				tag = 0;
				while ( d < p && '0' <= *d && *d <= '9' )
					tag = 10*tag + *d++ - '0';
				if ( !(recognized = d == p) ) /* all digits */
					tag = -1;
			}
			if ( ! recognized && s->dict ) { /* dict lookup */
				int i=s->dict->len, l = p-b;
				Field *f = s->dict->field;
				for ( ; i--; f++ )
					if ( l == f->len && memcmp( b, f->val, l ) ) {
						tag = f->tag;
						break;
					}
			}
			/* create new field unless it's a continuation line */
			if ( p == b && s->rec->len )
				;
			else if ( recognized )
				RADD( s->rec, tag, 0, 0, s->rec != s->buf );
			else
				RADD( s->rec, tag, b, p-b, s->rec != s->buf );
			if ( ! s->rec )
				goto outamem;
			/* did that */
			lio->b.done = (b = p) - lio->b.c;
			if ( LCS_ISFR(lcs_latin1_ct[*p]) )
				break;
			state = RS_SEP;
		} case RS_SEP:
			/* p = b is at beginning of separator */
			while ( p < e && ' ' == *p ) /* skip leading blanks */
				p++;
			if ( p < e && LCS_ISCST(lcs_latin1_ct[*p]) && '\t' != *p++ )
				while ( p < e && ' ' == *p ) /* skip trailing blanks */
					p++;
			if ( e == p ) {
				if ( 4096 > p - b )
					goto reload;
				sMsg( ERR_TRASH, "sep too long" );
				goto fatal;
			}
			assert( s->rec->len );
			if ( s->rec->field[s->rec->len - 1].len )
				RCAT( s->rec, b, p-b, s->rec != s->buf );
			if ( ! s->rec )
				goto outamem;
			lio->b.done = (b = p) - lio->b.c;
			if ( LCS_ISFR(lcs_latin1_ct[*p]) )
				break;
			state = RS_VAL;
		case RS_VAL: default: /* ??? */
			while ( p < e && !LCS_ISFR(lcs_latin1_ct[*p]) )
				p++;
			assert( s->rec->len );
			if ( p > b )
				RCAT( s->rec, b, p-b, s->rec != s->buf );
			if ( ! s->rec )
				goto outamem;
			lio->b.done = (b = p) - lio->b.c;
			if ( p == e )
				goto reload;
			/* else we found line or record separator */
		} /* switch state */
		if ( '\r' == *p ) { /* do that ole' ugly CR/NL quatsch */
			if ( e-p < 2 && 0 < LIO_FILL(lio) ) /* extend end */
				e = lio->b.c + lio->b.fill;
			if ( e-p < 2 )
				s->flg |= RS_EOL; /* possibly check later */
			else if ( '\n' == p[1] )
				p++;
		}
		lio->b.done = 1 + p - lio->b.c; /* consider p eaten */
		if ( LCS_R == lcs_latin1_ct[*p]
			|| (emptyline && (OPENISIS_STOPONEMPTY & s->flg))
		)
			break;
		state = RS_TAG;
	}
	/* got record */
	state = RS_REC;
	if ( s->rec && s->rec->len ) {
		if ( (OPENISIS_AUTOCLONE & s->flg) && s->rec == s->buf )
			s->rec = rDup( s->buf, 0, 0 );
		ret = 1;
	}
	goto done;
outamem:
fatal:
	if ( LIO_SISOPEN( s->in ) )
		LIO_CLOSE( s->in );
done:
	s->flg = state | (~3 & s->flg); /* save state */
	/* sMsg( LOG_ERROR, "fill %d done %d", lio->b.fill, lio->b.done ); */
	return ret;
}	/* openIsisReadStream */


int rSer ( char *buf, OpenIsisRec *rec )
{
	Field *f = rec->field;
	int i = rec->len;
	char *p = buf, *e;

	for ( ; i--; f++ ) {
		p += i2a( p, f->tag );
		*p++ = 9; /* TAB */
		memcpy( p, f->val, f->len );
		for ( e = p + f->len; p<e; )
			if ( 10 == *p++ )
				p[-1] = 11;
		*p++ = 10;
	}
	*p++ = 10;
	return p - buf;
}	/* rSer */


/** binary serialize, turns newline into newline tab
	buf must have len >= 2*rec->used
*/
int rSerB ( char *buf, OpenIsisRec *rec )
{
	Field *f = rec->field;
	int i = rec->len;
	char *p = buf;
	const char *s, *e;

	for ( ; i--; f++ ) {
		p += i2a( p, f->tag );
		*p++ = 9; /* TAB */
		for ( e = (s = f->val) + f->len; s<e; )
			if ( 10 == (*p++ = *s++) )
				*p++ = 9;
		*p++ = 10;
	}
	*p++ = 10;
	return p - buf;
}	/* rSerB */


char* rSerA (Rec *rec, char *buf, int *len) {
	if (!len) {
		return 0;
	}
	if (!rec) {
		*len = 0;
		return buf;
	}
	if (*len <= rec->used) {
		buf = (char*) mAlloc (1 + rec->used);
		if (!buf) {
			sMsg (ERR_NOMEM, "rSerA: cannot malloc %d", (1 + rec->used));
			return 0;
		}
	}
	*len = rSer (buf, rec);
	return buf;
}


int rDeser ( OpenIsisRec **rp, const char *buf, int len, int flg )
{
	Rec *rec = *rp;
	Field *f;
	const char *p = buf, *e = buf+len;
	char *q;
	int dis = RDIS & flg;
	int tag;
	/* non-transparent mode - no continuation lines */
	for ( ; e > p; p++ ) {
		if ( 10 == *p ) { /* blank line */
			if ( OPENISIS_STOPONEMPTY & flg )
				break;
			continue;
		}
		p += a2il( p, e-p, &tag );
		if ( e > p && 9 == *p ) /* TAB */
			p++;
		RADD( rec, tag, 0, e-p, dis || rec!=*rp );
		if ( !rec )
			break;
		f = rec->field + rec->len-1;
		for ( q=(char*)f->val; e>p && 10 != *p; )
			if ( 11 == (*q++ = *p++) )
				q[-1] = 10;
		rec->used += f->len = q - f->val;
	}
	*rp = rec;
	return p-buf;
}	/* rDeser */