hyperestraier/trunk/estseek.c

/*************************************************************************************************
 * A sample searcher of Hyper Estraier
 *                                                      Copyright (C) 2004-2005 Mikio Hirabayashi
 * This file is part of Hyper Estraier.
 * Hyper Estraier is free software; you can redistribute it and/or modify it under the terms of
 * the GNU Lesser General Public License as published by the Free Software Foundation; either
 * version 2.1 of the License or any later version.  Hyper Estraier is distributed in the hope
 * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 * License for more details.
 * You should have received a copy of the GNU Lesser General Public License along with Hyper
 * Estraier; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
 * Boston, MA 02111-1307 USA.
 *************************************************************************************************/


#if defined(MYFCGI)
#include <fcgi_stdio.h>
#endif
#include "estraier.h"
#include "myconf.h"

#define CONFSUFFIX     ".conf"           /* suffix of the configuration file */
#define KWDBNAME       "kwords"          /* name of the database for keywords */
#define DATTRLFILE     "_lfile"          /* name of the attribute of the local file name */
#define DATTRSCORE     "#score"          /* name of the pseudo-attribute of score */
#define NUMBUFSIZ      32                /* size of a buffer for a number */
#define OUTBUFSIZ      262144            /* size of the output buffer */
#define MINIBNUM       31                /* bucket number of map for trivial use */
#define LOCKRETRYNUM   16                /* number of retries when locking failure */
#define MISSRETRYNUM   3                 /* number of retries when missing documents */
#define MISSINCRATIO   8                 /* ratio of increment number when missing */
#define DEFPERPAGE     10                /* default number of show documents per page */
#define NAVIPAGES      10                /* number of pages in paging navigation */
#define SPCACHEMNUM    1048576           /* max number of the special cache */


/* global variables for configurations */
const char *g_conffile = NULL;           /* path of the configuration file */
const char *g_indexname = NULL;          /* name of the index */
const char *g_tmplfile = NULL;           /* path of the template file */
const char *g_topfile = NULL;            /* path of the top page file */
const char *g_logfile = NULL;            /* path of the log file */
const char *g_lprefix = NULL;            /* local prefix of the URI of each document */
const char *g_gprefix = NULL;            /* global prefix of the URI of each document */
const char *g_gsuffix = NULL;            /* global suffix of the URI of each document */
const char *g_dirindex = NULL;           /* name of the index file in a directory */
const CBLIST *g_replexprs = NULL;        /* list of URI replacement expressions */
const char *g_perpage = NULL;            /* CSV of numbers of shown documents per page */
int g_attrselect = FALSE;                /* whether to use select boxes for extension form */
int g_showscore = FALSE;                 /* whether to show scores */
const CBLIST *g_extattrs = NULL;         /* list of extra attributes of each document */
int g_snipwwidth = -1;                   /* whole width of the snippet */
int g_sniphwidth = -1;                   /* width of beginning of the text */
int g_snipawidth = -1;                   /* width around each highlighted word */
int g_condgstep = -1;                    /* step of N-gram */
int g_dotfidf = FALSE;                   /* whether to do TF-IDF tuning */
int g_smplphrase = FALSE;                /* whether to use simplefied phrase */
int g_candetail = FALSE;                 /* whether to show detail link */
int g_smlrvnum = -1;                     /* number of elements of a vecter for similarity */
const char *g_spcache = NULL;            /* name of the attribute of special cache */


/* global variables for parameters */
const char *p_phrase = NULL;             /* search phrase */
const char *p_attr = NULL;               /* narrowing attribute */
const char *p_attrval = NULL;            /* separated value of narrowing attribute */
const char *p_order = NULL;              /* ordering attribute */
int p_perpage = 0;                       /* number of show documents per page */
int p_pagenum = 0;                       /* number of the page */
int p_detail = 0;                        /* ID of the document to be detailed */
int p_similar = 0;                       /* ID of the seed document of similarity search */


/* other global variables */
char g_outbuf[OUTBUFSIZ];                /* output buffer */
const char *g_scriptname = NULL;         /* name of the script */
const char *g_tmpltext = NULL;           /* text of the template */
const char *g_toptext = NULL;            /* text of the top page */
ESTDB *g_db = NULL;                      /* main database object */
CURIA *g_kwdb = NULL;                    /* keyword database object */
double g_etime = 0.0;                    /* elepsed time */
int g_tabidx = 0;                        /* counter of tab indexes */


/* function prototypes */
int main(int argc, char **argv);
static int realmain(int argc, char **argv);
static void showerror(const char *msg);
static const char *skiplabel(const char *str);
static CBMAP *getparameters(void);
static void myestdbclose(ESTDB *db);
static void xmlprintf(const char *format, ...);
static CBMAP *vectorizer(void *db, int id, void *kwdb);
static void setsimilarphrase(void);
static void showpage(void);
static void showform(void);
static void showtop(void);
static void showresult(ESTDOC **docs, int dnum, CBMAP *hints, int miss);
static void showdoc(ESTDOC *doc, const CBLIST *words, CBMAP *cnames, int detail);
static char *makeshownuri(const char *uri);
static void showinfo(void);
static void outputlog(void);


/* main routine */
int main(int argc, char **argv){
#if defined(MYFCGI)
  static int cnt = 0;
  while(FCGI_Accept() >= 0){
    if(++cnt >= 256){
      cbggcsweep();
      g_db = NULL;
      g_kwdb = NULL;
      cnt = 0;
    }
    p_phrase = NULL;
    p_attr = NULL;
    p_attrval = NULL;
    p_order = NULL;
    p_perpage = 0;
    p_pagenum = 0;
    p_detail = 0;
    p_similar = 0;
    realmain(argc, argv);
  }
  return 0;
#else
  return realmain(argc, argv);
#endif
}


/* real main routine */
static int realmain(int argc, char **argv){
  CBLIST *lines, *rlist, *alist;
  CBMAP *params;
  const char *rp;
  char *tmp, *wp;
  int i, ecode;
  /* set configurations */
  cbstdiobin();
  setvbuf(stdout, g_outbuf, _IOFBF, OUTBUFSIZ);
  g_scriptname = argv[0];
  if((rp = getenv("SCRIPT_NAME")) != NULL) g_scriptname = rp;
  if((rp = strrchr(g_scriptname, '/')) != NULL) g_scriptname = rp + 1;
  tmp = cbmalloc(strlen(g_scriptname) + strlen(CONFSUFFIX) + 1);
  sprintf(tmp, "%s", g_scriptname);
  cbglobalgc(tmp, free);
  if(!(wp = strrchr(tmp, '.'))) wp = tmp + strlen(tmp);
  sprintf(wp, "%s", CONFSUFFIX);
  g_conffile = tmp;
  if(!(lines = cbreadlines(g_conffile))) showerror("the configuration file is missing.");
  cbglobalgc(lines, (void (*)(void *))cblistclose);
  rlist = cblistopen();
  cbglobalgc(rlist, (void (*)(void *))cblistclose);
  alist = cblistopen();
  cbglobalgc(alist, (void (*)(void *))cblistclose);
  for(i = 0; i < cblistnum(lines); i++){
    rp = cblistval(lines, i, NULL);
    if(cbstrfwimatch(rp, "indexname:")){
      g_indexname = skiplabel(rp);
    } else if(cbstrfwimatch(rp, "tmplfile:")){
      g_tmplfile = skiplabel(rp);
    } else if(cbstrfwimatch(rp, "topfile:")){
      g_topfile = skiplabel(rp);
    } else if(cbstrfwimatch(rp, "logfile:")){
      g_logfile = skiplabel(rp);
    } else if(cbstrfwimatch(rp, "lprefix:")){
      g_lprefix = skiplabel(rp);
    } else if(cbstrfwimatch(rp, "gprefix:")){
      g_gprefix = skiplabel(rp);
    } else if(cbstrfwimatch(rp, "gsuffix:")){
      g_gsuffix = skiplabel(rp);
    } else if(cbstrfwimatch(rp, "dirindex:")){
      g_dirindex = skiplabel(rp);
    } else if(cbstrfwimatch(rp, "replace:")){
      cblistpush(rlist, skiplabel(rp), -1);
    } else if(cbstrfwimatch(rp, "perpage:")){
      g_perpage = skiplabel(rp);
    } else if(cbstrfwimatch(rp, "attrselect:")){
      if(!cbstricmp(skiplabel(rp), "true")) g_attrselect = TRUE;
    } else if(cbstrfwimatch(rp, "showscore:")){
      if(!cbstricmp(skiplabel(rp), "true")) g_showscore = TRUE;
    } else if(cbstrfwimatch(rp, "extattr:")){
      cblistpush(alist, skiplabel(rp), -1);
    } else if(cbstrfwimatch(rp, "snipwwidth:")){
      g_snipwwidth = atoi(skiplabel(rp));
    } else if(cbstrfwimatch(rp, "sniphwidth:")){
      g_sniphwidth = atoi(skiplabel(rp));
    } else if(cbstrfwimatch(rp, "snipawidth:")){
      g_snipawidth = atoi(skiplabel(rp));
    } else if(cbstrfwimatch(rp, "condgstep:")){
      g_condgstep = atoi(skiplabel(rp));
    } else if(cbstrfwimatch(rp, "dotfidf:")){
      if(!cbstricmp(skiplabel(rp), "true")) g_dotfidf = TRUE;
    } else if(cbstrfwimatch(rp, "smplphrase:")){
      if(!cbstricmp(skiplabel(rp), "true")) g_smplphrase = TRUE;
    } else if(cbstrfwimatch(rp, "candetail:")){
      if(!cbstricmp(skiplabel(rp), "true")) g_candetail = TRUE;
    } else if(cbstrfwimatch(rp, "smlrvnum:")){
      g_smlrvnum = atoi(skiplabel(rp));
    } else if(cbstrfwimatch(rp, "spcache:")){
      g_spcache = skiplabel(rp);
    }
  }
  if(!g_indexname) showerror("indexname is undefined.");
  if(!g_tmplfile) showerror("tmplfile is undefined.");
  if(!g_topfile) showerror("topfile is undefined.");
  if(!g_logfile) showerror("logfile is undefined.");
  if(!g_lprefix) showerror("lprefix is undefined.");
  if(!g_gprefix) showerror("gprefix is undefined.");
  if(!g_gsuffix) showerror("gsuffix is undefined.");
  if(!g_dirindex) showerror("dirindex is undefined.");
  g_replexprs = rlist;
  if(!g_perpage) showerror("perpage is undefined.");
  g_extattrs = alist;
  if(g_snipwwidth < 0) showerror("snipwwidth is undefined.");
  if(g_sniphwidth < 0) showerror("sniphwidth is undefined.");
  if(g_snipawidth < 0) showerror("snipawidth is undefined.");
  if(g_condgstep < 1) showerror("condgstep is undefined.");
  if(!g_spcache) showerror("spcache is undefined.");
  /* read parameters */
  params = getparameters();
  cbglobalgc(params, (void (*)(void *))cbmapclose);
  if(!(p_phrase = cbmapget(params, "phrase", -1, NULL))) p_phrase = "";
  while(*p_phrase == ' ' || *p_phrase == '\t'){
    p_phrase++;
  }
  if(!(p_attr = cbmapget(params, "attr", -1, NULL))) p_attr = "";
  while(*p_attr == ' ' || *p_attr == '\t'){
    p_attr++;
  }
  if(!(p_attrval = cbmapget(params, "attrval", -1, NULL))) p_attrval = "";
  while(*p_attrval == ' ' || *p_attrval == '\t'){
    p_attrval++;
  }
  if(cbstrfwmatch(p_attr, "gstep=")){
    g_condgstep = atoi(p_attr + 6);
    p_attr = "";
  }
  if(cbstrfwmatch(p_attr, "tfidf=")){
    g_dotfidf = !cbstricmp(p_attr + 6, "true");
    p_attr = "";
  }
  if(!(p_order = cbmapget(params, "order", -1, NULL))) p_order = "";
  while(*p_order == ' ' || *p_order == '\t'){
    p_order++;
  }
  if((rp = cbmapget(params, "perpage", -1, NULL)) != NULL) p_perpage = atoi(rp);
  if(p_perpage < 1) p_perpage = DEFPERPAGE;
  if((rp = cbmapget(params, "detail", -1, NULL)) != NULL) p_detail = atoi(rp);
  if(p_detail < 1) p_detail = 0;
  if((rp = cbmapget(params, "similar", -1, NULL)) != NULL) p_similar = atoi(rp);
  if(p_similar < 1) p_similar = 0;
  if((rp = cbmapget(params, "pagenum", -1, NULL)) != NULL) p_pagenum = atoi(rp);
  if(p_pagenum < 1) p_pagenum = 1;
  if((rp = cbmapget(params, "enc", -1, NULL)) != NULL){
    if((tmp = est_iconv(p_phrase, -1, rp, "UTF-8", NULL, NULL)) != NULL){
      p_phrase = tmp;
      cbglobalgc(tmp, free);
    }
    if((tmp = est_iconv(p_attr, -1, rp, "UTF-8", NULL, NULL)) != NULL){
      p_attr = tmp;
      cbglobalgc(tmp, free);
    }
    if((tmp = est_iconv(p_attrval, -1, rp, "UTF-8", NULL, NULL)) != NULL){
      p_attrval = tmp;
      cbglobalgc(tmp, free);
    }
    if((tmp = est_iconv(p_order, -1, rp, "UTF-8", NULL, NULL)) != NULL){
      p_order = tmp;
      cbglobalgc(tmp, free);
    }
  }
  /* read the other files and the database */
  if(!g_db){
    if(!(tmp = cbreadfile(g_tmplfile, NULL))) showerror("the template file is missing.");
    cbglobalgc(tmp, free);
    g_tmpltext = tmp;
    if(!(tmp = cbreadfile(g_topfile, NULL))) showerror("the top page file is missing.");
    cbglobalgc(tmp, free);
    g_toptext = tmp;
    for(i = 0; i <= LOCKRETRYNUM; i++){
      if((g_db = est_db_open(g_indexname, ESTDBREADER | ESTDBLCKNB, &ecode)) != NULL) break;
      if(ecode != ESTELOCK) showerror("the index is missing or broken.");
      est_usleep(1000 * 1000);
    }
    if(!g_db) showerror("the index is being updated now.");
    cbglobalgc(g_db, (void (*)(void *))myestdbclose);
    if(g_spcache[0] != '\0') est_db_set_special_cache(g_db, g_spcache, SPCACHEMNUM);
  }
  setsimilarphrase();
  /* show the page */
  showpage();
  /* output the log message */
  outputlog();
  return 0;
}


/* show the error page and exit */
static void showerror(const char *msg){
  printf("Status: 500 Internal Server Error\r\n");
  printf("Content-Type: text/plain; charset=UTF-8\r\n");
  printf("\r\n");
  printf("Error: %s\n", msg);
  exit(1);
}


/* skip the label of a line */
static const char *skiplabel(const char *str){
  if(!(str = strchr(str, ':'))) return "";
  str++;
  while(*str != '\0' && (*str == ' ' || *str == '\t')){
    str++;
  }
  return str;
}


/* get CGI parameters */
static CBMAP *getparameters(void){
  int maxlen = 1024 * 1024 * 32;
  CBMAP *map, *attrs;
  CBLIST *pairs, *parts;
  const char *rp, *body;
  char *buf, *key, *val, *dkey, *dval, *wp, *bound, *fbuf, *aname;
  int i, len, c, blen, flen;
  map = cbmapopenex(37);
  buf = NULL;
  len = 0;
  if((rp = getenv("REQUEST_METHOD")) != NULL && !strcmp(rp, "POST") &&
     (rp = getenv("CONTENT_LENGTH")) != NULL && (len = atoi(rp)) > 0){
    if(len > maxlen) len = maxlen;
    buf = cbmalloc(len + 1);
    for(i = 0; i < len && (c = getchar()) != EOF; i++){
      buf[i] = c;
    }
    buf[i] = '\0';
    if(i != len){
      free(buf);
      buf = NULL;
    }
  } else if((rp = getenv("QUERY_STRING")) != NULL){
    buf = cbmemdup(rp, -1);
    len = strlen(buf);
  }
  if(buf && len > 0){
    if((rp = getenv("CONTENT_TYPE")) != NULL && cbstrfwmatch(rp, "multipart/form-data") &&
       (rp = strstr(rp, "boundary=")) != NULL){
      rp += 9;
      bound = cbmemdup(rp, -1);
      if((wp = strchr(bound, ';')) != NULL) *wp = '\0';
      parts = cbmimeparts(buf, len, bound);
      for(i = 0; i < cblistnum(parts); i++){
        body = cblistval(parts, i, &blen);
        attrs = cbmapopen();
        fbuf = cbmimebreak(body, blen, attrs, &flen);
        if((rp = cbmapget(attrs, "NAME", -1, NULL)) != NULL){
          cbmapput(map, rp, -1, fbuf, flen, FALSE);
          aname = cbsprintf("%s-filename", rp);
          if((rp = cbmapget(attrs, "FILENAME", -1, NULL)) != NULL)
            cbmapput(map, aname, -1, rp, -1, FALSE);
          free(aname);
        }
        free(fbuf);
        cbmapclose(attrs);
      }
      cblistclose(parts);
      free(bound);
    } else {
      pairs = cbsplit(buf, -1, "&");
      for(i = 0; i < cblistnum(pairs); i++){
        key = cbmemdup(cblistval(pairs, i, NULL), -1);
        if((val = strchr(key, '=')) != NULL){
          *(val++) = '\0';
          dkey = cburldecode(key, NULL);
          dval = cburldecode(val, NULL);
          cbmapput(map, dkey, -1, dval, -1, FALSE);
          free(dval);
          free(dkey);
        }
        free(key);
      }
      cblistclose(pairs);
    }
  }
  free(buf);
  return map;
}


/* close the database */
static void myestdbclose(ESTDB *db){
  int ecode;
  est_db_close(db, &ecode);
}


/* output escaped string */
static void xmlprintf(const char *format, ...){
  va_list ap;
  char *tmp, cbuf[32];
  unsigned char c;
  int cblen;
  va_start(ap, format);
  while(*format != '\0'){
    if(*format == '%'){
      cbuf[0] = '%';
      cblen = 1;
      format++;
      while(strchr("0123456789 .+-", *format) && *format != '\0' && cblen < 31){
        cbuf[cblen++] = *format;
        format++;
      }
      cbuf[cblen++] = *format;
      cbuf[cblen] = '\0';
      switch(*format){
      case 's':
        tmp = va_arg(ap, char *);
        if(!tmp) tmp = "(null)";
        printf(cbuf, tmp);
        break;
      case 'd':
        printf(cbuf, va_arg(ap, int));
        break;
      case 'o': case 'u': case 'x': case 'X': case 'c':
        printf(cbuf, va_arg(ap, unsigned int));
        break;
      case 'e': case 'E': case 'f': case 'g': case 'G':
        printf(cbuf, va_arg(ap, double));
        break;
      case '@':
        tmp = va_arg(ap, char *);
        if(!tmp) tmp = "(null)";
        while(*tmp){
          switch(*tmp){
          case '&': printf("&amp;"); break;
          case '<': printf("&lt;"); break;
          case '>': printf("&gt;"); break;
          case '"': printf("&quot;"); break;
          default:
            if(!((*tmp >= 0 && *tmp <= 0x8) || (*tmp >= 0x0e && *tmp <= 0x1f))) putchar(*tmp);
            break;
          }
          tmp++;
        }
        break;
      case '?':
        tmp = va_arg(ap, char *);
        if(!tmp) tmp = "(null)";
        while(*tmp){
          c = *(unsigned char *)tmp;
          if((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
             (c >= '0' && c <= '9') || (c != '\0' && strchr("_-.", c))){
            putchar(c);
          } else {
            printf("%%%02X", c);
          }
          tmp++;
        }
        break;
      case '%':
        putchar('%');
        break;
      }
    } else {
      putchar(*format);
    }
    format++;
  }
  va_end(ap);
}


/* create a vector of keywords */
static CBMAP *vectorizer(void *db, int id, void *kwdb){
  CBMAP *kwords;
  char *mbuf;
  int msiz;
  if(!(mbuf = crget((CURIA *)kwdb, (char *)&id, sizeof(int), 0, -1, &msiz))) return NULL;
  kwords = cbmapload(mbuf, msiz);
  free(mbuf);
  return kwords;
}


/* set the phrase for similarity search */
static void setsimilarphrase(void){
  ESTDOC *doc;
  CBMAP *svmap;
  CBDATUM *datum;
  const char *kbuf, *vbuf;
  char *ptr;
  int ksiz, vsiz;
  if(!cbstrfwimatch(p_phrase, ESTOPSIMILAR) && p_similar < 1) return;
  if(g_smlrvnum < 1){
    p_phrase = "";
    return;
  }
  if(!g_kwdb){
    ptr = cbsprintf("%s%c%s", g_indexname, ESTPATHCHR, KWDBNAME);
    if((g_kwdb = cropen(ptr, CR_OREADER, -1, -1)) != NULL){
      cbglobalgc(g_kwdb, (void (*)(void *))crclose);
      est_db_set_vectorizer(g_db, vectorizer, g_kwdb);
    }
    free(ptr);
  }
  if(p_similar < 1) return;
  svmap = g_kwdb ? vectorizer(g_db, p_similar, g_kwdb) : NULL;
  if(!svmap && (doc = est_db_get_doc(g_db, p_similar, 0)) != NULL){
    svmap = est_db_etch_doc(g_dotfidf ? g_db : NULL, doc, g_smlrvnum);
    est_doc_delete(doc);
  } else if(!svmap){
    return;
  }
  datum = cbdatumopen(ESTOPSIMILAR, -1);
  cbmapiterinit(svmap);
  while((kbuf = cbmapiternext(svmap, &ksiz)) != NULL){
    vbuf = cbmapget(svmap, kbuf, ksiz, &vsiz);
    cbdatumcat(datum, " WITH ", -1);
    cbdatumcat(datum, vbuf, vsiz);
    cbdatumcat(datum, " ", 1);
    cbdatumcat(datum, kbuf, ksiz);
  }
  ptr = cbdatumtomalloc(datum, NULL);
  cbglobalgc(ptr, free);
  p_phrase = ptr;
  cbmapclose(svmap);
}


/* show the page */
static void showpage(void){
  ESTCOND *cond;
  ESTDOC **docs;
  CBMAP *hints;
  CBLIST *elems;
  const char *rp;
  char *tmp, numbuf[NUMBUFSIZ];
  int i, tnum, max, *res, rnum, sc, dnum, miss;
  printf("Cache-Control: no-cache, must-revalidate, no-transform\r\n");
  printf("Pragma: no-cache\r\n");
  printf("Content-Disposition: inline; filename=%s\r\n", g_scriptname);
  printf("Content-Type: text/html; charset=UTF-8\r\n");
  printf("\r\n");
  g_etime = est_gettimeofday();
  cond = est_cond_new();
  if(p_phrase[0] != '\0') est_cond_set_phrase(cond, p_phrase);
  if(p_attr[0] != '\0'){
    if(p_attrval[0] != '\0'){
      tmp = cbsprintf("%s %s", p_attr, p_attrval);
      est_cond_add_attr(cond, tmp);
      free(tmp);
    } else {
      est_cond_add_attr(cond, p_attr);
    }
  }
  if(p_order[0] != '\0') est_cond_set_order(cond, p_order);
  switch(g_condgstep){
  case 1:
    est_cond_set_options(cond, ESTCONDSURE);
    break;
  case 2:
    est_cond_set_options(cond, ESTCONDUSU);
    break;
  case 3:
    est_cond_set_options(cond, ESTCONDFAST);
    break;
  case 4:
    est_cond_set_options(cond, ESTCONDAGIT);
    break;
  }
  if(!g_dotfidf) est_cond_set_options(cond, ESTCONDNOIDF);
  if(g_smplphrase) est_cond_set_options(cond, ESTCONDSIMPLE);
  if(g_showscore) est_cond_set_options(cond, ESTCONDSCFB);
  tnum = 0;
  max = p_pagenum * p_perpage * 1.3 + 1;
  do {
    est_cond_set_max(cond, max);
    hints = cbmapopenex(MINIBNUM);
    res = est_db_search(g_db, cond, &rnum, hints);
    if(g_candetail && p_detail > 0){
      if(rnum < 1) cbmapput(hints, "", 0, "1", 1, TRUE);
      free(res);
      res = cbmalloc(sizeof(int));
      res[0] = p_detail;
      rnum = 1;
    }
    docs = cbmalloc(rnum * sizeof(ESTDOC *) + 1);
    dnum = 0;
    miss = 0;
    for(i = 0; i < rnum; i++){
      if(!(docs[dnum] = est_db_get_doc(g_db, res[i], dnum < p_pagenum * p_perpage ? 0 :
                                       ESTGDNOATTR | ESTGDNOTEXT))){
        miss++;
        continue;
      }
      if((sc = est_cond_score(cond, i)) >= 0){
        sprintf(numbuf, "%d", sc);
        est_doc_add_attr(docs[dnum], DATTRSCORE, numbuf);
      }
      dnum++;
    }
    if(tnum <= MISSRETRYNUM && miss > 0 && max <= rnum && dnum < p_pagenum * p_perpage + 1){
      for(i = 0; i < dnum; i++){
        est_doc_delete(docs[i]);
      }
      free(docs);
      free(res);
      cbmapclose(hints);
      max *= MISSINCRATIO;
      tnum++;
      continue;
    }
    break;
  } while(TRUE);
  g_etime = est_gettimeofday() - g_etime;
  elems = cbxmlbreak(g_tmpltext, FALSE);
  for(i = 0; i < cblistnum(elems); i++){
    rp = cblistval(elems, i, NULL);
    if(!strcmp(rp, "<!--ESTFORM-->")){
      showform();
    } else if(!strcmp(rp, "<!--ESTRESULT-->")){
      if(p_phrase[0] == '\0' && p_attr[0] == '\0' && p_detail < 1){
        showtop();
      } else {
        showresult(docs, dnum, hints, miss);
      }
    } else if(!strcmp(rp, "<!--ESTINFO-->")){
      showinfo();
    } else {
      printf("%s", rp);
    }
  }
  for(i = 0; i < dnum; i++){
    est_doc_delete(docs[i]);
  }
  cblistclose(elems);
  free(docs);
  free(res);
  cbmapclose(hints);
  est_cond_delete(cond);
}


/* show the form */
static void showform(void){
  CBLIST *list;
  const char *elem;
  int i, num;
  xmlprintf("<div id=\"estform\" class=\"estform\">\n");
  xmlprintf("<form action=\"%@\" method=\"get\" id=\"form_self\">\n", g_scriptname);
  xmlprintf("<div class=\"form_basic\">\n");
  xmlprintf("<input type=\"text\" name=\"phrase\" value=\"%@\""
            " size=\"80\" id=\"phrase\" class=\"text\" tabindex=\"%d\" accesskey=\"0\" />\n",
            p_phrase, ++g_tabidx);
  xmlprintf("<input type=\"submit\" value=\"Search\""
            " id=\"search\" class=\"submit\" tabindex=\"%d\" accesskey=\"1\" />\n",
            ++g_tabidx);
  xmlprintf("</div>\n");
  xmlprintf("<div class=\"form_extension\">\n");
  xmlprintf("<select name=\"perpage\" id=\"perpage\" tabindex=\"%d\">\n", ++g_tabidx);
  list = cbsplit(g_perpage, -1, ",");
  for(i = 0; i < cblistnum(list); i++){
    elem = cblistval(list, i, NULL);
    if(elem[0] == '\0') continue;
    num = atoi(elem);
    xmlprintf("<option value=\"%d\"%s>%d</option>\n",
              num, num == p_perpage ? " selected=\"selected\"" : "", num);
  }
  cblistclose(list);
  xmlprintf("</select>\n");
  xmlprintf("per page, with\n");
  if(g_attrselect){
    xmlprintf("<select name=\"attr\" id=\"attr\" tabindex=\"%d\">\n", ++g_tabidx);
    xmlprintf("<option value=\"\">--</option>\n");
    xmlprintf("<option value=\"@title ISTRINC\"%s>title including</option>\n",
              cbstrfwmatch(p_attr, "@title ISTRINC") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@title ISTRBW\"%s>title beginning with</option>\n",
              cbstrfwmatch(p_attr, "@title ISTRBW") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@title ISTREW\"%s>title ending with</option>\n",
              cbstrfwmatch(p_attr, "@title ISTREW") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@author ISTRINC\"%s>author including</option>\n",
              cbstrfwmatch(p_attr, "@author ISTRINC") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@author ISTRBW\"%s>author beginning with</option>\n",
              cbstrfwmatch(p_attr, "@author ISTRBW") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@author ISTREW\"%s>author ending with</option>\n",
              cbstrfwmatch(p_attr, "@author ISTREW") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@mdate NUMLT\"%s>date less than</option>\n",
              cbstrfwmatch(p_attr, "@mdate NUMLT") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@mdate NUMGE\"%s>date not less than</option>\n",
              cbstrfwmatch(p_attr, "@mdate NUMGE") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@size NUMLT\"%s>size less than</option>\n",
              cbstrfwmatch(p_attr, "@size NUMLT") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@size NUMGE\"%s>size not less than</option>\n",
              cbstrfwmatch(p_attr, "@size NUMGE") ? " selected=\"selected\"" : "");
    xmlprintf("</select>\n");
    xmlprintf("<input type=\"text\" name=\"attrval\" value=\"%@\""
              " size=\"16\" id=\"attrval\" class=\"text\" tabindex=\"%d\" accesskey=\"2\" />\n",
              p_attrval, ++g_tabidx);
    xmlprintf(", order by\n");
    xmlprintf("<select name=\"order\" id=\"order\" tabindex=\"%d\">\n", ++g_tabidx);
    xmlprintf("<option value=\"\">score</option>\n");
    xmlprintf("<option value=\"@title STRA\"%s>title (asc)</option>\n",
              !strcmp(p_order, "@title STRA") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@title STRD\"%s>title (desc)</option>\n",
              !strcmp(p_order, "@title STRD") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@author STRA\"%s>author (asc)</option>\n",
              !strcmp(p_order, "@author STRA") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@author STRD\"%s>author (desc)</option>\n",
              !strcmp(p_order, "@author STRD") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@mdate NUMA\"%s>date (asc)</option>\n",
              !strcmp(p_order, "@mdate NUMA") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@mdate NUMD\"%s>date (desc)</option>\n",
              !strcmp(p_order, "@mdate NUMD") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@size NUMA\"%s>size (asc)</option>\n",
              !strcmp(p_order, "@size NUMA") ? " selected=\"selected\"" : "");
    xmlprintf("<option value=\"@size NUMD\"%s>size (desc)</option>\n",
              !strcmp(p_order, "@size NUMD") ? " selected=\"selected\"" : "");
    xmlprintf("</select>\n");
  } else {
    xmlprintf("<input type=\"text\" name=\"attr\" value=\"%@\""
              " size=\"24\" id=\"attr\" class=\"text\" tabindex=\"%d\" accesskey=\"2\" />\n",
              p_attr, ++g_tabidx);
    xmlprintf(", order by\n");
    xmlprintf("<input type=\"text\" name=\"order\" value=\"%@\""
              " size=\"24\" id=\"order\" class=\"text\" tabindex=\"%d\" accesskey=\"3\" />\n",
              p_order, ++g_tabidx);
  }
  xmlprintf("</div>\n");
  xmlprintf("</form>\n");
  xmlprintf("</div>\n");
}


/* show the top message */
static void showtop(void){
  printf("%s", g_toptext);
}


/* show the result */
static void showresult(ESTDOC **docs, int dnum, CBMAP *hints, int miss){
  CBMAP *cnames;
  CBLIST *words;
  const char *key, *myphrase;
  char cname[NUMBUFSIZ];
  int i, hits, snum, start, end, cnum, pnum;
  xmlprintf("<div id=\"estresult\" class=\"estresult\">\n");
  hits = atoi(cbmapget(hints, "", 0, NULL)) - miss;
  start = (p_pagenum - 1) * p_perpage;
  end = p_pagenum * p_perpage;
  if(end > dnum) end = dnum;
  xmlprintf("<div class=\"resinfo\">");
  xmlprintf("Results of <strong>%d</strong> - <strong>%d</strong>",
            start + (hits > 0 ? 1 : 0), end);
  xmlprintf(" of about <strong>%d</strong>", hits);
  if(p_phrase[0] != '\0' && strlen(p_phrase) < 128)
    xmlprintf(" for <strong>%@</strong>", p_phrase);
  if(g_etime > 0.0) xmlprintf(" (%.3f sec.)", g_etime / 1000.0);
  if(miss > p_perpage * p_pagenum) xmlprintf("*");
  xmlprintf("</div>\n");
  if(cbmaprnum(hints) > 2 || (p_phrase[0] != '\0' && p_attr[0] != '\0')){
    xmlprintf("<div class=\"hints\">");
    cbmapiterinit(hints);
    i = 0;
    while((key = cbmapiternext(hints, NULL)) != NULL){
      if(key[0] == '\0') continue;
      if(i++ > 0) xmlprintf(", ");
      xmlprintf("<span class=\"hword\">%s (%s)</span>", key, cbmapget(hints, key, -1, NULL));
    }
    xmlprintf("</div>\n");
  }
  words = cblistopen();
  cbmapiterinit(hints);
  while((key = cbmapiternext(hints, NULL)) != NULL){
    if(key[0] == '\0' || atoi(cbmapget(hints, key, -1, NULL)) < 0) continue;
    cblistpush(words, key, -1);
  }
  cnames = cbmapopenex(MINIBNUM);
  cnum = 0;
  for(i = 0; i < cblistnum(words); i++){
    sprintf(cname, "key%d", ++cnum);
    cbmapput(cnames, cblistval(words, i, NULL), -1, cname, -1, FALSE);
  }
  for(snum = start; snum < end; snum++){
    showdoc(docs[snum], words, cnames, g_candetail && p_detail > 0);
  }
  cbmapclose(cnames);
  cblistclose(words);
  if(dnum < 1) xmlprintf("<p class=\"note\">Your search did not match any documents.</p>\n");
  myphrase = p_similar > 0 ? "" : p_phrase;
  xmlprintf("<div class=\"paging\">\n");
  if(p_pagenum > 1){
    xmlprintf("<a href=\"%@?phrase=%?&amp;attr=%?&amp;attrval=%?&amp;order=%?"
              "&amp;perpage=%d&amp;pagenum=%d&amp;similar=%d\" class=\"navi\">PREV</a>\n",
              g_scriptname, myphrase, p_attr, p_attrval, p_order,
              p_perpage, p_pagenum - 1, p_similar);
  } else {
    xmlprintf("<span class=\"void\">PREV</span>\n");
  }
  pnum = (hits - 1 - (hits - 1) % p_perpage + p_perpage) / p_perpage;
  if(hits > 0 && p_detail < 1){
    for(i = p_pagenum > NAVIPAGES ? p_pagenum - NAVIPAGES + 1 : 1;
        i == 1 || (i <= pnum && i < p_pagenum + NAVIPAGES); i++){
      if(i == p_pagenum){
        printf("<span class=\"pnow\">%d</span>\n", i);
      } else {
        xmlprintf("<a href=\"%@?phrase=%?&amp;attr=%?&amp;attrval=%?&amp;order=%?"
                  "&amp;perpage=%d&amp;pagenum=%d&amp;similar=%d\" class=\"pnum\">%d</a>\n",
                  g_scriptname, myphrase, p_attr, p_attrval, p_order, p_perpage, i, p_similar, i);
      }
    }
  }
  if(snum < dnum){
    xmlprintf("<a href=\"%@?phrase=%?&amp;attr=%?&amp;attrval=%?&amp;order=%?"
              "&amp;perpage=%d&amp;pagenum=%d&amp;similar=%d\" class=\"navi\">NEXT</a>\n",
              g_scriptname, myphrase, p_attr, p_attrval, p_order,
              p_perpage, p_pagenum + 1, p_similar);
  } else {
    xmlprintf("<span class=\"void\">NEXT</span>\n");
  }
  xmlprintf("</div>\n");
  xmlprintf("</div>\n");
}


/* show a document */
static void showdoc(ESTDOC *doc, const CBLIST *words, CBMAP *cnames, int detail){
  CBMAP *kwords;
  CBLIST *names, *lines;
  const char *uri, *title, *score, *val, *name, *line, *cname;
  char *turi, *tsv, *pv, *str;
  int i, id;
  id = est_doc_id(doc);
  if(!(uri = est_doc_attr(doc, ESTDATTRURI))) uri = ".";
  turi = makeshownuri(uri);
  if(!(title = est_doc_attr(doc, ESTDATTRTITLE))) title = "";
  if(title[0] == '\0' && !(title = est_doc_attr(doc, DATTRLFILE))) title = "";
  if(title[0] == '\0' && ((pv = strrchr(uri, '/')) != NULL)) title = pv + 1;
  if(title[0] == '\0') title = "(no title)";
  if(!(score = est_doc_attr(doc, DATTRSCORE))) score = "";
  xmlprintf("<dl class=\"doc\" id=\"doc_%d\">\n", id);
  xmlprintf("<dt>");
  xmlprintf("<a href=\"%@\" class=\"doc_title\">%@</a>", turi, title);
  if(score[0] != '\0') xmlprintf(" <span class=\"doc_score\">%@</span>", score);
  xmlprintf("</dt>\n");
  if(detail){
    names = est_doc_attr_names(doc);
    for(i = 0; i < cblistnum(names); i++){
      name = cblistval(names, i, NULL);
      if(name[0] != '_' && strcmp(name, ESTDATTRURI) && strcmp(name, ESTDATTRTITLE) &&
         (val = est_doc_attr(doc, name)) != NULL && val[0] != '\0'){
        xmlprintf("<dd class=\"doc_attr\">");
        xmlprintf("%@: <span class=\"doc_val\">%@</span>", name, val);
        xmlprintf("</dd>\n");
      }
    }
    cblistclose(names);
    if(g_smlrvnum > 0){
      xmlprintf("<dd class=\"doc_attr\">");
      xmlprintf("#vector: <span class=\"doc_val\">");
      kwords = est_db_etch_doc(g_db, doc, g_smlrvnum);
      cbmapiterinit(kwords);
      for(i = 0; (name = cbmapiternext(kwords, NULL)) != NULL; i++){
        if(i > 0) xmlprintf(", ");
        xmlprintf("%@ (%@)\n", name, cbmapget(kwords, name, -1, NULL));
      }
      cbmapclose(kwords);
      xmlprintf("</span>");
      xmlprintf("</dd>\n");
    }
  } else {
    for(i = 0; i < cblistnum(g_extattrs); i++){
      str = cbmemdup(cblistval(g_extattrs, i, NULL), -1);
      if((pv = strchr(str, '|')) != NULL){
        *pv = '\0';
        pv++;
        if((val = est_doc_attr(doc, str)) != NULL && val[0] != '\0'){
          xmlprintf("<dd class=\"doc_attr\">");
          xmlprintf("%@: <span class=\"doc_val\">%@</span>", pv, val);
          xmlprintf("</dd>\n");
        }
      }
      free(str);
    }
  }
  xmlprintf("<dd class=\"doc_text\">");
  tsv = est_doc_make_snippet(doc, words, detail ? INT_MAX : g_snipwwidth,
                             detail ? INT_MAX : g_sniphwidth, g_snipawidth);
  lines = cbsplit(tsv, -1, "\n");
  for(i = 0; i < cblistnum(lines); i++){
    line = cblistval(lines, i, NULL);
    if(line[0] == '\0'){
      if(i < cblistnum(lines) - 1) xmlprintf(" ... ");
    } else if((pv = strchr(line, '\t')) != NULL){
      str = cbmemdup(line, pv - line);
      if(!(cname = cbmapget(cnames, pv + 1, -1, NULL))) cname = "key0";
      xmlprintf("<strong class=\"key %@\">%@</strong>", cname, str);
      free(str);
    } else {
      xmlprintf("%@", line);
    }
  }
  cblistclose(lines);
  free(tsv);
  xmlprintf("</dd>\n");
  xmlprintf("<dd class=\"doc_navi\">\n");
  xmlprintf("<span class=\"doc_uri\">%@</span>\n", turi);
  if(g_candetail)
    xmlprintf("- <a href=\"%@?phrase=%?&amp;detail=%d&amp;perpage=%d\" class=\"detail\">"
              "[detail]</a>\n", g_scriptname, p_similar > 0 ? "" : p_phrase, id, p_perpage);
  if(g_smlrvnum > 0)
    xmlprintf("- <a href=\"%@?similar=%d&amp;perpage=%d\" class=\"similar\">[similar]</a>\n",
              g_scriptname, id, p_perpage);
  xmlprintf("</dd>\n");
  xmlprintf("</dl>\n");
  free(turi);
}


/* make a URI to be shown */
static char *makeshownuri(const char *uri){
  const char *prefix;
  char *turi, *file, *bef, *aft, *pv, *nuri, *wp;
  int i;
  if(cbstrfwimatch(uri, g_lprefix)) uri += strlen(g_lprefix);
  prefix = g_gprefix;
  if(cbstrfwimatch(uri, "file://") || cbstrfwimatch(uri, "ftp://") ||
     cbstrfwimatch(uri, "http://") || cbstrfwimatch(uri, "https://")) prefix = "";
  turi = cbsprintf("%s%s%s", prefix, uri, g_gsuffix);
  if(g_dirindex[0] != '\0' && (file = strrchr(turi, '/')) != NULL &&
     !cbstricmp(file + 1, g_dirindex)){
    file[1] = '\0';
  }
  for(i = 0; i < cblistnum(g_replexprs); i++){
    bef = cbmemdup(cblistval(g_replexprs, i, NULL), -1);
    if((pv = strstr(bef, "{{!}}")) != NULL){
      *pv = '\0';
      aft = pv + 5;
    } else {
      aft = "";
    }
    if((pv = strstr(turi, bef)) != NULL){
      nuri = cbmalloc(strlen(turi) + strlen(aft) + 1);
      wp = nuri;
      memcpy(wp, turi, pv - turi);
      wp += pv - turi;
      wp += sprintf(wp, "%s", aft);
      sprintf(wp, "%s", pv + strlen(bef));
      free(turi);
      turi = nuri;
    }
    free(bef);
  }
  return turi;
}


/* show the top */
static void showinfo(void){
  xmlprintf("<div id=\"estinfo\" class=\"estinfo\">");
  xmlprintf("Powered by Hyper Estraier %@, with %d documents and %d words.",
            est_version, est_db_doc_num(g_db), est_db_word_num(g_db));
  xmlprintf("</div>\n");
}


/* output the log message */
static void outputlog(void){
  FILE *ofp;
  const char *val;
  if(g_logfile[0] == '\0' || !(ofp = fopen(g_logfile, "ab"))) return;
  if(!(val = getenv("REMOTE_ADDR"))) val = "0.0.0.0";
  fprintf(ofp, "%s:", val);
  if(!(val = getenv("REMOTE_PORT"))) val = "0";
  fprintf(ofp, "%s\t", val);
  fprintf(ofp, "%s\t", p_phrase);
  if(!(val = getenv("HTTP_USER_AGENT"))) val = "*";
  fprintf(ofp, "%s\n", val);
  fclose(ofp);
}


/* END OF FILE */