0.9.9e/core/core.h

/*
        The Malete project - the Z39.2/Z39.50 database framework of OpenIsis.
        Version 0.9.x (patchlevel see file Version)
        Copyright (C) 2001-2003 by Erik Grziwotz, erik@openisis.org

        This library is free software; you can redistribute it and/or
        modify it under the terms of the GNU Lesser General Public
        License as published by the Free Software Foundation; either
        version 2.1 of the License, or (at your option) any later version.

        This library is distributed in the hope that it will be useful,
        but WITHOUT ANY WARRANTY; without even the implied warranty of
        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
        See the GNU Lesser General Public License for more details.

        You should have received a copy of the GNU Lesser General Public
        License along with this library; if not, write to the Free Software
        Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

        see README for more information
EOH */
#ifndef CORE_H

#include <assert.h>
#include <string.h> /* various str* and mem* */

/*
        $Id: core.h,v 1.43 2004/11/11 15:47:08 kripke Exp $
        full interface of the Malete core
*/

#define CORE_VERSION "0.9.9"


/* ****************************************************************** */
/*                                                                    */
/* COMPILER FEATURES                                                  */
/*                                                                    */
/* ****************************************************************** */

#ifndef __STDC__ /* defined by ISO C */
#error "need ANSI/ISO C compiler"
#endif
/* "cpp -dM </dev/null" lists nonstandard machine and OS macros */

/* optimizing */
#ifdef __GNUC__
#       define OPT_INLINE __inline__ /* OPT_INLINE type func () */
#else
#       define OPT_INLINE
#endif
#if defined( __i386__ ) && defined( __GNUC__ )
/* called function pops args; makes most functions slightly faster */
/* type OPT_STDCALL func () on both declaration and definition */
#       define OPT_STDCALL __attribute__((stdcall))
#else
#       define OPT_STDCALL
#endif
#if defined( __i386__ ) && defined( __GNUC__ ) && defined(NDEBUG)
/* more aggressive: up to 3 args in registers, fails with -pg */
/* type OPT_REGPARM func () in declaration is sufficient */
/* also using in definition allows fallback to stdcall */
#       define OPT_REGPARM __attribute__((regparm(3)))
#else
#       define OPT_REGPARM OPT_STDCALL
#endif

/* CPU features */
#if defined( __sparc__ ) || defined( __ppc__ )
#       define CPU_BIG_ENDIAN
#endif
#if defined( __sparc__ )
#       define CPU_NEED_ALIGN
#endif
/* max bits of PAGE_SIZE; min is always 12 */
/* grep -r 'fine PAGE_SHIFT' /usr/src/linux/include/asm-* : 12..16 */
#if defined( __ia64__ )
#       define CPU_PAGE_SHIFT 16
#elif defined( __arm__ )
#       define CPU_PAGE_SHIFT 15
#elif defined( __i386__ ) || defined( __ppc__ )
#       define CPU_PAGE_SHIFT 12 /* there are more not exceeding 12 bits */
#else /* however 13 bits is not too much waste */
#       define CPU_PAGE_SHIFT 13 /* sparc, sparc64, alpha, m68k, cris */
#endif

/* 64 bit integer */
#ifdef __GNUC__
#       define LOLO_BUILTIN long long
#       define LOLO( v ) v##LL
#       define LULU( v ) v##ULL
#elif defined( _MSC_VER ) || defined( __BORLANDC__ )
#       define LOLO_BUILTIN __int64
#       define LOLO( v ) v /* is there some special suffix ??? */
#       define LULU( v ) v
#elif 0xFFFFFFFFL != ~0L /* 64 bit compiler ? */
#       define LOLO_BUILTIN long
#       define LOLO( v ) v##L
#       define LULU( v ) v##UL
#else
#       error "please use a compiler providing a 64 bit integer type. thanks."
#endif
typedef LOLO_BUILTIN lolo;
typedef unsigned LOLO_BUILTIN lulu;


/* ****************************************************************** */
/*                                                                    */
/* BUILD FEATURES                                                     */
/*                                                                    */
/* ****************************************************************** */

/* support the ENV_SHARED mode */
#if defined(BUILD_SHMODE) && defined(WIN32)
#       undef BUILD_SHMODE
#endif


/* ****************************************************************** */
/*                                                                    */
/* DATA STRUCTURES AND UTILITIES                                      */
/*                                                                    */
/* ****************************************************************** */

/* some characters */
#define TAB 9 /* horizontal, that is */
#define LF 10 /* LineFeed a.k.a. newline - '\n' isn't really well defined */
#define VT 11 /* vertical, used as newline replacement */
#define CR 13 /* for windoze, telnet and the like */

/** similar to atoi/strtol, but
        - string needs not be 0 terminated (unless l < 0)
        - cares for hex 0x, but not octal 0.
*/
extern int a2i ( const char *p, int l );
/** similar to a2i but
        - return number of parsed characters
        - put result in *res
*/
extern int a2il ( const char *p, int l, int *res );
/**     similar to a2i but
        - return dflt if less than l characters of p has been successfully parsed
*/
extern int a2id ( const char *p, int l, int dflt );
/** print int (NOT 0 terminated !).
        p must have 11 bytes space
        @return number of chars (up to 10 digits + minus sign)
*/
extern int i2a ( char *p, int i );
extern int u2a ( char *p, unsigned u );
/* print exactly n digits, do not add 0 byte */
extern void u2a0 ( char *p, unsigned u, unsigned n );

extern const char b36dig[36]; /* '0'..'9', 'a'..'z' */
/* 0..35 '0'..'9', 'a'..'z' and 'A'..'Z', 36 else */
extern const char b36val[256];

extern const unsigned char lat1up[256]; /* latin 1 uppercase */
extern const unsigned char lat1ct[256]; /* latin 1 ctype */
enum { /* character type bits */
        CT_WHITE = 0x01, /* all <= ' ' */
        CT_DIGIT = 0x02, /* 0..9 */
        CT_ALPHA = 0x04, /* 'A'..'Z','a'..'z' */
        CT_IDENT = 0x08, /* digits,alphas,underscore */
        CT_SPECL = 0x10, /* all other ASCIIs */
        CT_LATIN = 0x20  /* ident + non ASCII Latin1 alphas */
};
enum { /* character type values */
        CT_W = CT_WHITE,
        CT_D = CT_DIGIT|CT_IDENT|CT_LATIN,
        CT_A = CT_ALPHA|CT_IDENT|CT_LATIN,
        CT_I = CT_IDENT|CT_LATIN,
        CT_S = CT_SPECL,
        CT_L = CT_LATIN,
        CT_N = 0 /* other C1 control or symbol */
};
#define CT_IS(type, c) (CT_##type == lat1ct[(unsigned char)(c)])
#define CT_HAS(flg, c) (CT_##flg & lat1ct[(unsigned char)(c)])


/** replace from by to in bytes */
extern void mTr ( char *bytes, char from, char to, int len );


typedef struct Fld {
        int      tag;
        unsigned len; /* must use at most 31 bits, highest is temporarily abused */
        char    *val; /* not necessarily 0 terminated */
} Fld;

/*
        v (value) functions on single field
*/
#define V2I( f ) a2i( (f)->val, (f)->len )

/* field has primary value key length l */
#define VKEY( f, k, l ) ( \
        (f)->len >= (l) && !memcmp( (f)->val, k, l ) \
        && ((f)->len == (l) || TAB == (f)->val[(l)]) \
)

/* dup a field as 0-terminated string */
#define VDUPZ( f ) ((char*)mDupz((f)->val, (f)->len))

/*
        snip tab-separated subfields from value,
        setting tag to the subfield identifier and len/val to the contents.
        if opt is not 0, it lists the options wanted (* for the primary).
        dst->val should be 0-initialized;
        if it's greater than src->val, search starts at dst->val+dst->len,
        @return 0 if not found, else 1+dst->len
*/
extern int vGet ( Fld *dst, const Fld *src, const char *opt );
/*
        access to the primary value never fails, if initialized, so return len
*/
#define VPRI( dst, src ) ((dst)->val = 0, vGet(dst, src, 0), (dst)->len)
#define VEQZ( f, str ) (!strncmp((f)->val,str,(f)->len) && !str[(f)->len])

/**
        undo the encoding of lBin (see below).
        return #bytes int dst, which is <= src->len
extern int vDecod ( char *dst, const Fld *src );
*/

/* sign(a - b) */
extern int vCmp ( const Fld *a, const Fld *b );
/* a > b */
extern int vGt ( const Fld *a, const Fld *b );

/* comparision function type for rSort */
typedef int VGt ( const Fld *a, const Fld *b );


/*
        r (record) functions on an array of fields
        they expect the tag of the first field to be the negative number of fields.
        The value of the first field ("header") may contain various meta info.
        For lists representing database "records",
        the canonical format is [no[@pos]][<TAB>leader].
        Leader can be used e.g. to hold a Z39.2 leader as used by MARC.
        Other (protocol) lists should start with some type identifier.

        it's a matter of taste whether you like and use this typedef.
        as it's not going to be changed but only here to clarify the intend,
        the core functions do not use it.
*/
typedef const Fld *Rec;

#define RLEN( r )  (-(r)->tag)
#define REND( r )  ((r)-(r)->tag)
#define RLAST( r ) (REND(r)-1)

extern unsigned rSiz ( const Fld *r );

/**
        get occurence of field.
        @param pos if given, the first occ starting at pos is searched
                and pos is set to one after the found position (or after end).
                may be used to loop all on an int var initialized to 0.
        @return the field or 0
*/
extern const Fld *rGet ( const Fld *r, int tag, int *pos );

/*
        get field with given tag and primary value or empty pv
*/
extern const Fld *rKey ( const Fld *r, int tag, const char *key );

/*
        create a new const rec in a contigous mAlloced peace of mem
        if siz is 0, rSiz() is used
*/
extern const Fld *rDup ( const Fld *src, unsigned siz );


/** flatten (serialize) record
        to tag\tval lines ended by a blank line.
        buf must be of size rSiz(rec)
        + 13*RLEN(rec) for sign+10digits+tab+nl
        + 1 for the blank line
        @return # of bytes written
*/
extern int rSer ( char *buf, const Fld *rec );

/** sort fields
        WARNING: this cuts fields longer than 2GB !!!
        NOTE that rSort, unlike qsort, does NOT use a 3-way cmp function,
        but a boolean gt function
*/
extern void rSort ( Fld *rec, VGt *gt );
extern void rSortTag ( Fld *rec );
extern void rSortVal ( Fld *rec ); /* == rSort(vGt) */


#define DEFBLKLEN 8000 /* default buffer block length */
#define DEFFIELDS 40 /* default number of fields */

typedef struct LBlk { /* chained buffer block */
        struct LBlk *nxt;
        unsigned     siz;
        char         byt[DEFBLKLEN]; /* actual len may vary */
} LBlk;


/*
        A full-fledged, modifiable list.
*/
typedef struct List {
        Fld      *fld; /* fields list, initially fl0 */
        unsigned  fav; /* fields available at end of *fld buffer */
        int       siz; /* used secondary buffer size minus holes (add buf-blk.byt) */
        char     *buf; /* pointing into blk->byt */
        char     *end; /* of blk->byt */
        LBlk     *blk; /* buffer chain, initially bl0 */
        char     *bok; /* if == buf, buffers contain the serialization */
        Fld       fl0[DEFFIELDS];
        LBlk      bl0;
} List;


#define LLEN( l )  RLEN((l)->fld)
#define LEND( l )  REND((l)->fld)
#define LLAST( l ) RLAST((l)->fld)


/*
        initialize list and set header.
        if fmt is 0, no header is printed, and the list left empty.
        (i.e. the first field added will become the header).
        @return l
*/
extern List *lInit ( List *l, const char *fmt, ... );
/*
        completely clear all fields and buffers.
*/
extern List *lClr ( List *l );
/*
        clear all fields and buffers, but keep the header.
*/
extern List *lReset ( List *l );
/*
        clear all fields and buffers, keep nothing
*/
extern void OPT_STDCALL lFini ( List *l );

/*
        increase available space.
        If buffer has a flusher, call it.
        fields -1 means we want to append (keep last field contigous, do NOT flush).
        fields >0 means reserve at least mode fields.
*/
extern int OPT_REGPARM lExtend ( List *l, unsigned need, int fields );


/* bytes available at buf */
#define LAVL( l ) ((unsigned)((l)->end - (l)->buf))
#define LSIZ( l ) ((unsigned)((l)->buf - (l)->blk->byt + (l)->siz))

/*
        fragment used to start a new field with tag t.
        If the list was empty, the field will become the header,
        ignoring the tag.
*/
#define LDEFNEWF( l, t ) \
        int __i = (l)->fld->tag; \
        Fld *__f = (l)->fld - __i--; \
        assert(0 > __i); \
        __f->tag = t; \
        (l)->fld->tag = __i;

/* add field tag t reserving n bytes space */
#define LNEWF( l, t, n ) do { \
        if ( ((l)->fav && LAVL(l) >= n) || lExtend( l, n, 1 ) ) { \
                LDEFNEWF( l, t ) \
                __f->val = (l)->buf; \
                __f->len = 0; \
                (l)->fav --; \
        } } while(0)

/* add field tag t using n bytes prefilled space
        must be preextended for one field
*/
#define LPREF( l, t, n ) do { \
        if ( (l)->fav ) { \
                LDEFNEWF( l, t ) \
                __f->val = (l)->buf; \
                __f->len = n; \
                (l)->fav --; \
                (l)->buf += n; \
        } } while(0)


/* add field tag t with value v n bytes long */
#define LADD( l, t, v, n ) do { \
        if ( ((l)->fav && LAVL(l) >= n) || lExtend( l, n, 1 ) ) { \
                LDEFNEWF( l, t ) \
                memcpy( __f->val = (l)->buf, v, __f->len = n ); \
                (l)->fav --; \
                (l)->buf += n; \
        } } while(0)

/* append value v n bytes long */
#define LAPP( l, v, n ) do { \
        if ( LAVL(l) >= n || lExtend( l, n, -1 ) ) { \
                Fld *__f = LLAST(l); \
                memcpy( (l)->buf, v, n ); \
                __f->len += n; \
                (l)->buf += n; \
        } } while(0)

/* add field tag t with int value i */
#define LADDI( l, t, i ) do { \
        if ( ((l)->fav && LAVL(l) >= 12) || lExtend( l, 12, 1 ) ) { \
                LDEFNEWF( l, t ) \
                (l)->buf += __f->len = i2a( __f->val = (l)->buf, i ); \
                (l)->fav --; \
        } } while(0)

#define LADDS( l, t, s ) LADD( l, t, s, strlen(s) )
#define LAPPS( l, s )    LAPP( l, s, strlen(s) )
#define LADDF( l, f )    LADD( l, f->tag, f->val, f->len )


/**
        add a field with arbitrary binary data.
        The encoded field value will not contain newlines.
        The encoding maps a VT to two bytes VT 0,
        and a LF to VT 1, if the LF is followed by a 0 or 1, a single VT else.
extern void lBin ( List *l, int tag, const char *bytes, int len );
*/


/*
        append canonical options fields.
        args starting with '-' are treated as options and appended as
        tab separated subfield w/o the '-'.
        If the first arg does not start with '-', it is used as primary value.
        Other non-options are appended as subfields indicated by '@'.
        In order to skip the program's name, from main() use with argc-1,argv+1.
*/
extern int lArgv ( List *l, int tag, int argc, const char **argv );
/*
        same using 0-terminated list.
extern int lArgs ( List *l, int tag, const char *arg, ... );
*/
/**
        A variation on this theme as used for env.var:
        an initial non-option as command and following options go to the header.
        every following non-option starts a new field.
*/
extern List *lVar ( List *l, int argc, const char **argv );

/* if siz is 0, rSiz() is used */
extern int lCpy ( List *l, const Fld *src, unsigned siz );
#define LCPY( l, src ) lCpy( l, (src)->fld, LSIZ(src) )

/* create canned version of list as new record */
#define LCAN( l ) rDup( (l)->fld, LSIZ(l) )


/*
        Print a field to list.
        a fmt of 0 indicates varargs.
*/
extern int lOut ( List *to, int tag, const char *fmt, ... );


/*
        set the first field with tag to val (of len, -1 for strlen)
        kill additional fields with tag
        if no field found, add one at end
extern int lSet ( List *l, int tag, const char *val, unsigned len );
*/


enum { /* list parse state */
        LPS_SOR, /* 0, at start of record */
        LPS_SOL = 0x1000000, /* at start of line */
        LPS_TAG = 0x2000000, /* in tag (only digits seen in line) */
        LPS_VAL = 0x3000000, /* in val (some non-digit seen) */
        LPS_NEG = 0x4000000, /* found '-' at start of line */
        LPS_CR  = 0x8000000, /* kill carriage return character */
        LPS_LEN = 0x0ffffff
};
/**
        add serialized "text" to rec
        lps is the buffer length (of up to 16MB) + state bits
        @return new state, if buffer was exhausted,
                or #remaining bytes (|LPS_SOR), if a blank line was seen
*/
extern int lParse (List *l, const char *txt, int lps);


/**
        record sink.
        The sink function may be called with eor 0 after adding one or several fields
        to optionally flush partial records.
        It must be called with eor when a record is complete.
        In that case it must prepare List to receive a new header.
        This typically is done by flushing and resetting list,
        however, it would be sufficient to prepare for a new embedded
        subrecord by recording the position of the next head.
*/
typedef struct Sink Sink;
typedef void sink (Sink *that, int eor); /* clean all or part of buffer(s) */

struct Sink {
        List  lst;
        sink *snk; /* sinking function */
        void *dst; /* destination */
        int   off; /* start of current record in list, negative after partial flush */
};

#define SINK(s) (s)->snk(s, 0)
#define SEOR(s) (s)->snk(s, 1)
#define SCPY(s, l) do { LCPY(&(s)->lst, l); SEOR(s); } while (0)
#define SCPYR(s, r) do { lCpy(&(s)->lst, r, 0); SEOR(s); } while (0)

/* ****************************************************************** */
/*                                                                    */
/* ENVIRONMENT AND SYSTEM                                             */
/*                                                                    */
/* ****************************************************************** */

#ifndef WIN32
typedef int file;
#define FIL_NONE -1
#else
typedef void *file;
#define FIL_NONE 0
#endif

/* environment.
*/
typedef struct Env {
        List     *opt; /* options (header is packed command line) */
        file      in;
        Sink     *out; /* a proper record sink expecting complete records */
        Sink     *err; /* a usually field buffered sink for eOut messages */
        int       log; /* level */
        int       flg;
        int       wri; /* write mode none/excl/shared */
        int       sig; /* interrupted by signal */
        unsigned  psz; /* system page size = 1<<psh enforced min 4K */
        int       psh; /* page shift (bits) 12..CPU_PAGE_SHIFT */
        unsigned  rml; /* r memory map limit (max pages per pointer map) */
        unsigned  qml; /* q memory map limit (max pages per tree map) */
        /* LBlk     *blk[5]; for memory management */
} Env;

enum { /* env flags */
        ENV_BUFFER = 1, /* env.err is buffered */
        ENV_MSYNC = 2 /* use msync */
};
enum { /* env writing mode */
        ENV_RO,    /* readonly - not writing */
        ENV_EXCL  /* exclusive access (the default) */
#ifdef BUILD_SHMODE
        , ENV_SHARED /* shared access */
#endif
};
enum { /* signal code */
        ENV_CANCEL = 1, /* abort current request */
        ENV_TERM /* abort current request and exit */
};

extern Env env; /* main environment */

/**
        error and loglevel codes
*/
enum { /* our very own errno */
        ERR_OK    = 0, /* 0 is no error, also read(2)'s EINTR, EAGAIN */
        ERR_NO    = -1, /* get details from errno */
        /* error level FATAL: we are wrong */
        ERR_IDIOT = -2, /* caught programming error */
        ERR_TRASH = -3, /* database internal consistency */
        LOG_FATAL   = ERR_TRASH, /* fatal internal errors: we can't go on */
        /* error levels SYERR,ERROR,IOERR: system or user was wrong */
        ERR_NOMEM = -4, /* out of memory, also open(2)'s EMFILE, ENFILE */
        ERR_IO    = -5, /* real IO error, also write(2)'s ENOSPC, EPIPE */
        ERR_BUSY  = -6, /* object is busy */
        LOG_SYSERR  = ERR_BUSY, /* problem with system ressources: bad file, no mem */
        ERR_BADF  = -7, /* bad file, also read(2)'s EINVAL, some of open(2) */
        ERR_FAULT = -8, /* 0 pointer or bad sized buffer given */
        ERR_INVAL = -9, /* general invalid parameters, any EINVAL errno */
        LOG_ERROR   = ERR_INVAL, /* unusable input, database or query */
        ERR_AGAIN = -10, /* no data at intr or nonblock */
        LOG_IOERR   = ERR_AGAIN, /* problem on IO */
        LOG_WARN    = -11, /* bad input */
        /* logging levels: nothing was wrong */
        LOG_INFO    = -12, /* some major event like opening a db */
        LOG_VERBOSE = -13, /* any event like reading a record */
        LOG_DEBUG   = -14, /* lots of processing details (debug built only) */
        LOG_TRACE   = -15, /* database content (log_str) */
        LOG_ALL     = LOG_TRACE
};


#define EADD(t, v, n) LADD(&env.out->lst, t, v, n)
#define EAPP(v, n)    LAPP(&env.out->lst, v, n)
#define EADDS(t, s)   LADDS(&env.out->lst, t, s)
#define EAPPS(s)      LAPPS(&env.out->lst, s)
#define EADDI(t, i)   LADDI(&env.out->lst, t, i)
#define EADDF(f)      LADDF(&env.out->lst, f)

/*
        Append a field with abs(tag) to env.out for non-negative tags,
        else to env.err if tag >= env's loglevel.
        For system errors, additional info is appended.
        env.err's eof is called, if any.
        returns tag

        supports only a small subset of printf formats -- see the src
        has %b (bytes), which is like %.*s, but prints the string in hex
*/
extern int eOut ( int tag, const char *fmt, ... );
extern int eRr ( int tag, const char *fmt, ... );
#ifndef NDEBUG
# define LOG_DBG eRr
#elif defined( __GNUC__ )
# define LOG_DBG( args... )
#else
# define LOG_DBG (void) /* compiler should dispose statement off */
#endif

/**
        Init the env.
        The first field specifies general command and options.
        Following fields describe databases.
        If no sinks are provided, file sinks on stdout and stderr are used.
*/
extern void cInit ( List *args, Sink *out, Sink *err );
/* typical usage from main() */
#define CINIT(argl) cInit(lVar(lInit(argl,0), argc-1, argv+1), 0, 0)


/**
        memory management.
        All memory will be initialized,
        and all allocs but TryAlloc exit the process when out of memory.
        DO NOT MIX with other alloc/free routines.
*/
extern void *mAlloc ( int size );
extern void *mDup ( const void *str, int size );
/* cp siz bytes, append a 0 byte */
extern char *mDupz ( const char *str, int size );
extern void *mTryAlloc ( int size );
extern void  mFree ( void *mem );
extern Fld  *mFldAlloc ( int nfields );
extern void  mFldFree ( Fld *fld );
extern LBlk *mBlkAlloc ( int size );
extern void  mBlkFree ( LBlk *blk );
extern List *mListAlloc ( const char *name ); /* lInitialized with name */
extern void  mListFree ( List *l );
#define mFldAlloc(n) ((Fld*)mAlloc((n)*sizeof(Fld)))
#define mFldFree mFree
#define mBlkFree mFree
#define MFREE(p) do { mFree(p); (p)=0; } while(0)


/**
        set tm to current time, return difference in millis
*/
extern int tUpd ( lolo *tm );
/**
        print generalized time yyyyMMddHHmmss + 0 byte to buffer
        if tm is 0, current time is used
        if *tm is 0, *tm is updated
        buffer must have 15 bytes
        return millis
*/
extern int tGtf ( char *buf, lolo *tm );
/**
        grok the fine manual.
        like tGtf, but with additional 3 digits millis
        return buffer, which must have 18 bytes
*/
extern char *tGtfm ( char *buf, lolo *tm );
/**
        nanosl
*/
extern void tSleep ( lolo tm );


/* ************************************************************
        disk files (block devices)
*/


enum {
        /* basic open flags */
        FIL_RD    = 0x001, /* shall be opened for input */
        FIL_WR    = 0x002, /* shall be opened for output */
        FIL_RDWR  = 0x003, /* shall be opened for both */
        FIL_TRY   = 0x004, /* do not complain if open fails */
        /* write flags */
        FIL_CREAT = 0x010, /* shall be created */
        FIL_TRUNC = 0x020, /* shall be truncated */
        FIL_SYNC  = 0x040, /* syncing output */
        /* lock flags */
        FIL_TLOCK = 0x100, /* try locking (EX with WR) */
        FIL_BLOCK = 0x200, /* blocking lock (EX with WR) */
        FIL_FLOCK = 0x300 /* any locking is set */
};

/** open a new fid based on name and flags.
        TLOCK can be specified on any plattform,
        translating to a fcntl full file lock on *nix and a share mode on win
        @return 0 or some error code
*/
extern int fOpen ( file *f, const char *name, int flags );
extern int fClose ( file *f );
extern int fSize ( file f );
extern unsigned fTime ( file f ); /* mtime sec */


/*
        Like the syscalls, this returns the number of bytes on success, 0 on eof.
        fPwrite repeats and does not return an error when interrupted.
        On error, a negative value is returned.
*/
extern int fRead ( file *f, void *buf, unsigned len );
extern int fWrite ( file *f, const void *buf, unsigned len );
extern int fPread ( file *f, void *buf, unsigned len, unsigned off );
extern int fPwrite ( file *f, const void *buf, unsigned len, unsigned off );
extern int fSeek ( file *f, unsigned off );
extern int fTrunc ( file *f, unsigned length );
#ifdef BUILD_SHMODE
/* remove a full file lock as set by fOpen */
extern void fUnlock ( file f );
/*
        lock byte n
        use TLOCK or BLOCK, possibly with WR, to lock, 0 to unlock.
*/
extern int fLock ( file f, unsigned n, int flg );
#define FLOCK(f,n,flg) (ENV_SHARED==env.wri && fLock(f,n,flg))
#else
#define FLOCK(f,n,flg) 0
#endif
#define FLOCKSH(f,n) FLOCK(f,n,FIL_BLOCK)
#define FLOCKEX(f,n) FLOCK(f,n,FIL_BLOCK|FIL_WR)
#define FLOCKUN(f,n) FLOCK(f,n,0)

/** slurp in a whole file at once.
        @param buf points to buffer of size sz.
                lio_slurp will allocate one, if *buf is NULL
        @param sz maximum number of bytes to read
        @param name of file to slurp
        @param opt if != 0, do not complain on failure
        @return number of bytes read or negative on error
*/
extern int fSlurp ( char **buf, int sz, const char *name, int opt );

/* record-oriented sink to (file)dst.
        Expects the list to contain a proper record.
        If the header starts with a digit, a W<TAB> is prepended.
        An empty header is ommited.
*/
extern void fSinkr (Sink *that, int eor);
/* line-oriented sink (field values only) to (file)dst.
        leaves the header alone, ignores tags and blank values.
        yet prints a blank line on eor.
*/
extern void fSinkl (Sink *that, int eor);

/** a potentially mapped file */
typedef struct {
        file     fil;
        int      flg;
        char    *map;
        unsigned npg; /* in pages of env.psz */
        unsigned lim; /* max pages to map */
#ifdef WIN32
        char    *nam; /* for shared mapping */
        void    *hdl; /* "mapping object" */
#endif
} FMap;

/**
        open a file to be mapped.
        Like fOpen, but saves flags for later reference.
*/
extern int fMOpen ( FMap *fm, const char *name, int flags );
extern int fMClose ( FMap *fm );
/**
        map, remap or unmap a memory mapping

        @param fm the filemap
                members fil and flg must be set as of fOpen.
                if fm was mapped, the existing mapping is unmapped (or remapped)
        @param npg number of pages to map; 0 for no new mapping
        @return mapped length; if <= 0, fm->map is set to 0, else to memory region
*/
extern int fMap ( FMap *fm, unsigned npg );
/** sync a mapped page */
extern int fMSync ( FMap *fm, unsigned page );


/*
        file input buffer structure for fGets.
        suitable for both temp fix buffers and, with some care, for List buffers.
*/
typedef struct {
        file      f; /* file to read from */
        unsigned  n; /* current line number (1 based) */
        unsigned  o; /* offset of p from file start */
        char     *b; /* buffer base (const) */
        unsigned  s; /* buffer size (const) */
        char     *p; /* start of current line */
        unsigned  l; /* line len */
        unsigned  m; /* more bytes after p+l (including the LF) */
} FBuf;
/* sloppy but convenient initializer macro */
#define FIL_BUF( fb, fil, buf )  do { \
        fb.f = fil; fb.b = fb.p = buf; fb.s = sizeof(buf); \
        fb.n = 1; fb.o = fb.l = fb.m = 0; \
        } while (0)
/* even more convenient macro, must be end of data def */
#define FIL_DEFBUF( f ) \
        char buf[0x2000]; \
        FBuf fb; \
        FIL_BUF(fb, f, buf);
/*
        set p and l to next line, lines are terminated by LF.
        if l, advance p after current line.
        if m, skip next byte.
        if m, search for newline.
        if no newline found, mv p downto b and read more chars upto s.
        if l==s, buffer is exhausted (and m is 0).
        else if m, p[l] is a newline.
        else if FIL_NONE == f,
        no bytes could be read (an eof was seen or somebody set NBLOCK).
        return whether we got a line
*/
extern int fGets ( FBuf *fb );
extern int fGetr ( List *l, FBuf *fb );


/* ****************************************************************** */
/*                                                                    */
/* DATABASE                                                           */
/*                                                                    */
/* ****************************************************************** */

enum { /* flags for record and query data/index file pairs */
        DX_OPEN  = 0x1, /* open */
        DX_WRITE = 0x2, /* open for writing */
        DX_ASYNC = 0x4, /* no synced write */
        DX_MODIF = 0x8  /* modified */
};
/**
        record data and index.
        While this is accessible standalone, a logical database table
        may consist of several Rdx, e.g. per every million records.
*/
typedef struct Rdx {
        file        mrd;
        FMap        mrx;
        int         flg;
        int         mid; /* in records == maxid */
        int         rdl; /* length of data file in bytes */
        int         ptl; /* pointer bytes, by now always 8 */
        int         typ; /* type of pointer file */
} Rdx;

/** initialise from an already open fd.  */
extern int rInit ( Rdx *rdx );
/** flush and release any cache.  */
extern void rFini ( Rdx *rdx );

/*
        read record rid.
        if mpos and the rec is found at mpos or higher,
        rRead will backtrack to earlier versions.
*/
extern int rRead ( List *l, Rdx *rdx, int rid, unsigned mpos );
/*
        write a record
        use rid 0 to get new rid
        use opos -1 if you don't care about old pos
        to be transparent here, the rid@pos as found in standard recs
        is ignored and should be pre-skipped up to the leader.
        specify the record size if known, with 0, rSiz will be used
        @return the record id written (> 0) on success, <= 0 on error
*/
extern int rWrite ( Rdx *rdx, const Fld *r, int rid, int opos, unsigned siz );


/**
        query data and index
        leaf blocks have a configurable size from 512 bytes to 8K
        fork (inner node) blocks have always pagesize, i.e. 4K up to 64K

        While this is accessible standalone, a logical database table
        may consist of several Qdx, each holding some range of keys.
*/
typedef struct Cdx Cdx;

typedef struct Qdx { /* actually it's a B-Link-Tree */
        file           mqd; /* the leaves file */
        FMap           mqx; /* the tree file */
        const Cdx     *cdx; /* the collation */
        int            flg; /* flags: writeable */
        unsigned char  typ; /* cfg: leaf block type */
        unsigned char  ksz; /* cfg: max key length, default 255 */
        unsigned char  ptr; /* cfg: inverted file pointer type or plain value size */
        unsigned char  let; /* cfg: pct free on load */
        /* members set automatically:  */
        unsigned char  vsz; /* value size, min 4, default 8 */
        unsigned char  uni; /* value unique length (see qSet) */
        unsigned char  ftp; /* fork block type */
        unsigned char  dpt; /* depth (level of root over bottom > 0) */
        unsigned       lsz; /* leaf block size computed from type */
        unsigned       lln; /* # leaf blocks in index */
        unsigned       fln; /* # fork blocks in index */
        /* members considered internal:  */
        struct QLoad  *qld;
} Qdx;

enum { /* btree block type, size, flg */
        QDX_TYPMSK = 0xC0, /* highest 2 bits: basic type */
        QDX_LEAF   = 0x00, /* leaf block, portable */
        QDX_FORKLE = 0x40, /* fork block little endian */
        QDX_FORKBE = 0x80, /* fork block big endian */
        QDX_LEAFPV = 0xC0, /* leaf plain values (forks don't care) */
        /* next 2 bits 0x30 for future extensions */
        QDX_COMPRS = 0x08, /* flag compressed keys (not yet supported) */
        QDX_SIZMSK = 0x07, /* lowest 3 bits: blocksize */
        QDX_LEAF0K = 0x00, /* 1/2K blocks 0x0200 9+0 bits */
        QDX_LEAF1K = 0x01, /* 1K blocks 0x0400 */
        QDX_LEAF2K = 0x02, /* 2K blocks 0x0800 */
        QDX_LEAF4K = 0x03, /* 4K blocks 0x1000 */
        QDX_LEAF8K = 0x04  /* 8K blocks 0x2000, max for leaves */
};
enum {
        QDX_MAXVALPERLEAF = 0x800, /* max 8K / min 4 bytes vsz */
        QDX_LEAFSH =   9, /* leaf size shift+(0..4) ~ 512 bytes - 8K */
        QDX_FORKSH =  12  /* fork size shift+(0..4) ~ 4K - 64K */
};

typedef struct {
        unsigned char len;
        unsigned char byt[255];
}       Val;

typedef struct {
        Val           val;
        unsigned char len;
        unsigned char byt[255];
}       Key;


typedef struct QLoop QLoop;
/**
        callback for index loop.
        maybe called multiple times for same key, if it spans blocks.
        In this case, flag QSAME is set.
        loop stops if QCb returns != 0
*/
typedef int QCb ( QLoop *self );

struct QLoop {
        QCb   *qcb;
        Qdx   *qdx;
        int    flg;
        Key    key;
        Key    to;
        /* set on callback: */
        Key    cur;
        unsigned            nvals;
        const unsigned char *vals;
};

enum { /* flags */
        QLOOP = 0, /* loop endless */
        /* stop based on QLoop.key */
        QEQ   = 1, /* loop while == key */
        QPF   = 2, /* loop on prefix key */
        /* stop based on QLoop.to */
        QUPTO = 4, /* loop while < to */
        QINCL = 5, /* loop while <= to */
        QSTOP = 7, /* mask for stop mode */
        QSKIP = 8, /* skip the from key */
        QSAME = 0x10 /* callback on same key */
};


/** initialise from an already open fd.  */
extern int qInit ( Qdx *bt );
/** flush and release any cache.  */
extern void qFini ( Qdx *bt );

/*
        load a sorted series of keys and hits into index.
        call repeatedly, using a key with val.len 0 in last call
*/
extern int qLoad ( Qdx *bt, Key *key );
extern int qLoadf ( Qdx *bt, file *f );

/*
        write the key-value pair to the index and return 0, unless:
        - the value is zero on the 1st uni bytes
          and there is already such a value (unique key, return 3)
        - there is an all-zero value for the key (stopword, return 2)
        - it is already there (full duplicate found, return 1)
        With plain values, only full duplicates are checked.
        Uni is usually the length of the initial segment of value,
        which is the rid in fulltext mode, else the tag.
        Where a unique key is found, the value is copied to key.
*/
extern int qSet ( Qdx *bt, Key *key );
enum {
        QST_OK,
        QST_FOUND,
        QST_STPWRD,
        QST_UNIKEY
};
/*
        delete a key-value pair. return 1 if found, else 0
*/
extern int qDel ( Qdx *bt, Key *key );

extern int qLoop ( QLoop *self );


/*
        standard values structured as pointers
        denoting a record and position where key occurred.
        The value structure is up to 3 big endian unsigned numbers:
        0..2 bytes for tag,
        3+(0..3) bytes for rid (in fulltext mode: before the tag) and
        0..4 bytes for pos,
        totalling from 4 (vsz min.) to 12 bytes.
*/
typedef struct { /* where a key has a hit */
        unsigned short tag; /* while neg. tags are allowed, sorting is unsigned */
        unsigned short ext; /* extend row id to six bytes (used as db number) */
        unsigned       rid; /* row id */
        unsigned       pos; /* word pos, usually with field occ<<16 */
}       Ptr;

enum {
        QDX_TAGMSK = 0xC0, /* mask length of tag 0..2 */
        QDX_TAG1   = 0x40,
        QDX_TAG2   = 0x80,
        QDX_TAGSH  =    6, /* ... shifted by 6 */
        QDX_RIDMSK = 0x30, /* mask length of rid 0..3 */
        QDX_RIDMIN =    3, /* based on 3 */
        QDX_RIDSH  =    4, /* ... shifted by 4 */
        QDX_RID3   = 0x00,
        QDX_RID4   = 0x10,
        QDX_RID5   = 0x20,
        QDX_RID6   = 0x30,
        QDX_FULTXT = 0x08, /* traditional fulltext ordering */
        QDX_POSMSK = 0x07, /* mask length pos info 0..4 */
        /* default settings based on 3 byte rid */
        QDX_ISIS   = QDX_RID3|QDX_TAG2|QDX_FULTXT|3,
        /* 0x8B, 8 byte fulltext 3+2+3 IFP format */
        QDX_STDDB  = QDX_TAG1|QDX_RID3
        /* 0x40, 4 byte 3 rid + 1 for 256 field tags */
};

/* decode a value to a pointer */
extern void qRdVal ( Ptr *ptr, const unsigned char *val, unsigned char typ );
/* create a value from hit. */
extern void qMkVal ( Val *val, Ptr *ptr, unsigned char typ );
/* decode a key to plaintext */
extern int qRdKey ( Qdx *qdx, char *plain, int l, Key *key );
/* create a key from plaintext, truncating to ksz. */
extern void qMkKey ( Qdx *qdx, Key *key, char *b, int l );
/* create a key and value from line. */
extern int qMkKeyVal ( Qdx *qdx, Key *key, char *b, int l );


typedef struct {
        Qdx *qdx;
        int  del;
        Key  pfx;
        Ptr  ptr;
} QSet;
/*
        set or del one keys
        basically this behaves like qMkKey, qMkVal, qSet
        ptr.pos is incremented
        with pfx, pfx is prepended to the key (before qMkKey)
*/
extern int qSetKeyVal (QSet *qst, char *val, int len);
/*
        split the value into words and qSetKeyVal each
        return the number of entries made
*/
extern int qSetKeyVals (QSet *qst, char *val, int len);


/**
        collation
*/
#define CDX_MAXSEQ 15   /* max byte sequence length */


extern const Cdx *cOpen ( const Fld *src );
extern int cEnc ( const Cdx *cdx, Key *key, unsigned char *b, int l, int w );
extern int cDec ( const Cdx *cdx, unsigned char *b, int l, Key *key );


enum { /* see Metadata.txt */
        MET_OPT = 001,
        MET_UNU = 002,
        MET_CTP = 003,
        MET_COL = 004,
        MET_VER = 005,
        MET_FLD = 006
};

/**
        finally, a real database
*/
typedef struct Db {
        char       *nam;
        char       *pat;
        int         flg;
        int         mnt; /* mount count */
        struct Db  *nxt; /* linked list */
        const Fld  *opt; /* inner meta data */
        Rdx         rdx;
        Qdx         qdx;
} Db;

extern Db *dOpen (const char *dbname);
extern void dClose (Db *db);
extern void dCloseAll ();

/**
        access to record and query data via the db.
        In future versions these might become real functions taking
        care of multifile databases.
*/
#define dRead(l,db,rid) ((rid)?rRead(l,&(db)->rdx,rid,0):lCpy(l,(db)->opt,0))
#define dWrite(db,r,rid) rWrite(&(db)->rdx,r,rid,-1,0)

#define dSet(db,key) qSet(&(db)->qdx,key)
#define dDel(db,key) qDel(&(db)->qdx,key)
#define dLoop(db,ql) ((ql)->qdx=&(db)->qdx, qLoop(ql))

#define CORE_H
#endif /* CORE_H */