diff --git a/src/index/indexer.cpp b/src/index/indexer.cpp index f46c3f25..9b491755 100644 --- a/src/index/indexer.cpp +++ b/src/index/indexer.cpp @@ -1,5 +1,5 @@ #ifndef lint -static char rcsid[] = "@(#$Id: indexer.cpp,v 1.54 2007-05-21 09:00:28 dockes Exp $ (C) 2004 J.F.Dockes"; +static char rcsid[] = "@(#$Id: indexer.cpp,v 1.55 2007-05-22 07:40:00 dockes Exp $ (C) 2004 J.F.Dockes"; #endif /* * This program is free software; you can redistribute it and/or modify @@ -185,6 +185,11 @@ bool DbIndexer::init(bool resetbefore, bool rdonly) LOGERR(("DbIndexer: error opening database in %s\n", m_dbdir.c_str())); return false; } + + int idxflushmb; + if (m_config->getConfParam("idxflushmb", &idxflushmb)) + m_db.setFlushMb(idxflushmb); + return true; } diff --git a/src/rcldb/rcldb.cpp b/src/rcldb/rcldb.cpp index ed40e539..b0d05bb9 100644 --- a/src/rcldb/rcldb.cpp +++ b/src/rcldb/rcldb.cpp @@ -1,5 +1,5 @@ #ifndef lint -static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.108 2007-05-21 13:30:21 dockes Exp $ (C) 2004 J.F.Dockes"; +static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.109 2007-05-22 07:40:00 dockes Exp $ (C) 2004 J.F.Dockes"; #endif /* * This program is free software; you can redistribute it and/or modify @@ -498,7 +498,7 @@ string Native::makeAbstract(Xapian::docid docid, const list& iterms) Db::Db() : m_ndb(0), m_qOpts(QO_NONE), m_idxAbsTruncLen(250), m_synthAbsLen(250), - m_synthAbsWordCtxLen(4), m_mode(Db::DbRO) + m_synthAbsWordCtxLen(4), m_flushmb(-1), m_mode(Db::DbRO) { m_ndb = new Native(this); } @@ -1040,6 +1040,14 @@ bool Db::add(const string &fn, const Doc &idoc, return false; } } + if (m_flushmb > 0) { + m_curtxtsz += doc.text.length(); + if (m_curtxtsz / (1024*1024) >= m_flushmb) { + LOGDEB(("Db::add: text size >= %d Mb, flushing\n", m_flushmb)); + m_ndb->wdb.flush(); + m_curtxtsz = 0; + } + } return true; } diff --git a/src/rcldb/rcldb.h b/src/rcldb/rcldb.h index 4e45fa9a..5c8c5fd3 100644 --- a/src/rcldb/rcldb.h +++ b/src/rcldb/rcldb.h @@ -16,7 +16,7 @@ */ #ifndef _DB_H_INCLUDED_ #define _DB_H_INCLUDED_ -/* @(#$Id: rcldb.h,v 1.46 2007-02-02 10:10:53 dockes Exp $ (C) 2004 J.F.Dockes */ +/* @(#$Id: rcldb.h,v 1.47 2007-05-22 07:40:00 dockes Exp $ (C) 2004 J.F.Dockes */ #include #include @@ -174,6 +174,8 @@ class Db { bool filenameWildExp(const string& exp, list& names); string getReason(){return m_reason;} + /** Adjust flush threshold */ + void setFlushMb(int mb) {m_flushmb = mb;} private: @@ -200,6 +202,11 @@ private: // when building the abstract int m_synthAbsWordCtxLen; + // Flush threshold. Megabytes of text indexed before we flush. + int m_flushmb; + // Text bytes indexed since last flush + long long m_curtxtsz; + // Database directory string m_basedir;