tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

implemented adjustable indexing flush threshold

Browse Source
This commit is contained in:
dockes 2007-05-22 07:40:00 +00:00
parent e565fd4615
commit e04fbf3fb4
3 changed files with 24 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/index/indexer.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.54 2007-05-21 09:00:28 dockes Exp $ (C) 2004 J.F.Dockes"; static char rcsid[] = "@(#$Id: indexer.cpp,v 1.55 2007-05-22 07:40:00 dockes Exp $ (C) 2004 J.F.Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -185,6 +185,11 @@ bool DbIndexer::init(bool resetbefore, bool rdonly)
LOGERR(("DbIndexer: error opening database in %s\n", m_dbdir.c_str())); LOGERR(("DbIndexer: error opening database in %s\n", m_dbdir.c_str()));
return false; return false;
} }
int idxflushmb;
if (m_config->getConfParam("idxflushmb", &idxflushmb))
m_db.setFlushMb(idxflushmb);
return true; return true;
} }

12
src/rcldb/rcldb.cpp
View File

@ -1,5 +1,5 @@
#ifndef lint #ifndef lint
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.108 2007-05-21 13:30:21 dockes Exp $ (C) 2004 J.F.Dockes"; static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.109 2007-05-22 07:40:00 dockes Exp $ (C) 2004 J.F.Dockes";
#endif #endif
/* /*
* This program is free software; you can redistribute it and/or modify * This program is free software; you can redistribute it and/or modify
@ -498,7 +498,7 @@ string Native::makeAbstract(Xapian::docid docid, const list<string>& iterms)
Db::Db() Db::Db()
: m_ndb(0), m_qOpts(QO_NONE), m_idxAbsTruncLen(250), m_synthAbsLen(250), : m_ndb(0), m_qOpts(QO_NONE), m_idxAbsTruncLen(250), m_synthAbsLen(250),
m_synthAbsWordCtxLen(4), m_mode(Db::DbRO) m_synthAbsWordCtxLen(4), m_flushmb(-1), m_mode(Db::DbRO)
{ {
m_ndb = new Native(this); m_ndb = new Native(this);
} }
@ -1040,6 +1040,14 @@ bool Db::add(const string &fn, const Doc &idoc,
return false; return false;
} }
} }
if (m_flushmb > 0) {
m_curtxtsz += doc.text.length();
if (m_curtxtsz / (1024*1024) >= m_flushmb) {
LOGDEB(("Db::add: text size >= %d Mb, flushing\n", m_flushmb));
m_ndb->wdb.flush();
m_curtxtsz = 0;
}
}
return true; return true;
} }

9
src/rcldb/rcldb.h
View File

@ -16,7 +16,7 @@
*/ */
#ifndef _DB_H_INCLUDED_ #ifndef _DB_H_INCLUDED_
#define _DB_H_INCLUDED_ #define _DB_H_INCLUDED_
/* @(#$Id: rcldb.h,v 1.46 2007-02-02 10:10:53 dockes Exp $ (C) 2004 J.F.Dockes */ /* @(#$Id: rcldb.h,v 1.47 2007-05-22 07:40:00 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string> #include <string>
#include <list> #include <list>
@ -174,6 +174,8 @@ class Db {
bool filenameWildExp(const string& exp, list<string>& names); bool filenameWildExp(const string& exp, list<string>& names);
string getReason(){return m_reason;} string getReason(){return m_reason;}
/** Adjust flush threshold */
void setFlushMb(int mb) {m_flushmb = mb;}
private: private:
@ -200,6 +202,11 @@ private:
// when building the abstract // when building the abstract
int m_synthAbsWordCtxLen; int m_synthAbsWordCtxLen;
// Flush threshold. Megabytes of text indexed before we flush.
int m_flushmb;
// Text bytes indexed since last flush
long long m_curtxtsz;
// Database directory // Database directory
string m_basedir; string m_basedir;