tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

circache: append: resize dest to avoid recycling while appending. Clarifications

Browse Source
This commit is contained in:
Jean-Francois Dockes
2021-03-24 09:31:37 +01:00
parent 5e82af9798
commit 9eac638bb9
5 changed files with 144 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/circache.cpp
+66 -34
View File
@@ -14,6 +14,7 @@
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#define LOGGER_LOCAL_LOGINC 4
#include "autoconfig.h"
@@ -34,6 +35,7 @@
#include "chrono.h"
#include "zlibut.h"
#include "smallut.h"
#ifndef _WIN32
#include <sys/uio.h>
@@ -255,10 +257,8 @@ public:
return true;
}
void khDump() {
for (kh_type::const_iterator it = m_ofskh.begin();
it != m_ofskh.end(); it++) {
LOGDEB("Circache::KHDUMP: " << it->first.asHexString() << " " <<
it->second << "\n");
for (const auto& e : m_ofskh) {
LOGDEB("Circache::KHDUMP: " << e.first.asHexString() << " " << e.second << "\n");
}
}
@@ -270,8 +270,7 @@ public:
UdiH h(udi);
LOGDEB2("Circache::khFind: h " << h.asHexString() << " udi [" << udi <<
"]\n");
LOGDEB2("Circache::khFind: h " << h.asHexString() << " udi [" << udi << "]\n");
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
@@ -312,9 +311,8 @@ public:
}
// Clear entries for vector of udi/offs
bool khClear(const vector<pair<string, int64_t> >& udis) {
for (vector<pair<string, int64_t> >::const_iterator it = udis.begin();
it != udis.end(); it++) {
khClear(*it);
for (const auto& udioffs : udis) {
khClear(udioffs);
}
return true;
}
@@ -589,15 +587,13 @@ public:
return true;
}
bool readDicData(int64_t hoffs, EntryHeaderData& hd, string& dic,
string* data) {
bool readDicData(int64_t hoffs, EntryHeaderData& hd, string& dic, string* data) {
int64_t offs = hoffs + CIRCACHE_HEADER_SIZE;
// This syscall could be avoided in some cases if we saved the offset
// at each seek. In most cases, we just read the header and we are
// at the right position
if (lseek(m_fd, offs, 0) != offs) {
m_reason << "CirCache::get: lseek(" << offs << ") failed: " <<
errno;
m_reason << "CirCache::get: lseek(" << offs << ") failed: " << errno;
return false;
}
char *bf = 0;
@@ -863,9 +859,9 @@ public:
virtual status takeone(int64_t offs, const string& udi,
const EntryHeaderData& d) {
LOGDEB2("Circache:Scan: off " << offs << " udi [" << udi << "] dcsz " <<
d.dicsize << " dtsz " << d.datasize <<
" pdsz " << d.padsize << " flgs " << d.flags << "\n");
LOGDEB1("Circache:Scan: off " << offs << " udi [" << udi << "] dcsz " <<
d.dicsize << " dtsz " << d.datasize << " pdsz " << d.padsize <<
" flgs " << d.flags << " previnst " << m_instance << "\n");
if (!m_udi.compare(udi)) {
m_instance++;
m_offs = offs;
@@ -878,7 +874,6 @@ public:
}
};
// instance == -1 means get latest. Otherwise specify from 1+
bool CirCache::get(const string& udi, string& dic, string *data, int instance)
{
Chrono chron;
@@ -899,12 +894,11 @@ bool CirCache::get(const string& udi, string& dic, string *data, int instance)
int finst = 1;
EntryHeaderData d_good;
int64_t o_good = 0;
for (vector<int64_t>::iterator it = ofss.begin();
it != ofss.end(); it++) {
LOGDEB1("Circache::get: trying offs " << *it << "\n");
for (const auto& offset : ofss) {
LOGDEB1("Circache::get: trying offs " << offset << "\n");
EntryHeaderData d;
string fudi;
if (!m_d->readHUdi(*it, d, fudi)) {
if (!m_d->readHUdi(offset, d, fudi)) {
return false;
}
if (!fudi.compare(udi)) {
@@ -912,7 +906,7 @@ bool CirCache::get(const string& udi, string& dic, string *data, int instance)
// matches, else go on. If instance is -1 need to
// go to the end anyway
d_good = d;
o_good = *it;
o_good = offset;
if (finst == instance) {
break;
} else {
@@ -946,6 +940,10 @@ bool CirCache::get(const string& udi, string& dic, string *data, int instance)
return bret;
}
// It would be possible to have an option to only erase if this is the
// last entry in the file, by comparing the offsets from khFind() with
// m_oheadoffs. Read the last offset < m_oheadoffs and check that
// offset+sizes == oheadoffs
bool CirCache::erase(const string& udi, bool reallyclear)
{
if (m_d == 0) {
@@ -977,22 +975,22 @@ bool CirCache::erase(const string& udi, bool reallyclear)
return true;
}
for (vector<int64_t>::iterator it = ofss.begin(); it != ofss.end(); it++) {
LOGDEB2("CirCache::erase: reading at " << *it << "\n");
for (const auto& offset : ofss) {
LOGDEB2("CirCache::erase: reading at " << offset << "\n");
EntryHeaderData d;
string fudi;
if (!m_d->readHUdi(*it, d, fudi)) {
if (!m_d->readHUdi(offset, d, fudi)) {
return false;
}
LOGDEB2("CirCache::erase: found fudi [" << fudi << "]\n");
if (!fudi.compare(udi)) {
EntryHeaderData nd;
nd.padsize = d.dicsize + d.datasize + d.padsize;
LOGDEB2("CirCache::erase: rewrite at " << *it << "\n");
if (*it == m_d->m_nheadoffs) {
LOGDEB2("CirCache::erase: rewrite at " << offset << "\n");
if (offset == m_d->m_nheadoffs) {
m_d->m_npadsize = nd.padsize;
}
if (!m_d->writeEntryHeader(*it, nd, reallyclear)) {
if (!m_d->writeEntryHeader(offset, nd, reallyclear)) {
LOGERR("CirCache::erase: write header failed\n");
return false;
}
@@ -1339,26 +1337,60 @@ static bool copyall(std::shared_ptr<CirCache> occ,
return true;
}
// Append all entries from sdir to ddir
int CirCache::append(const string ddir, const string& sdir, string *reason)
int CirCache::appendCC(const string ddir, const string& sdir, string *reason)
{
ostringstream msg;
// Open source file
std::shared_ptr<CirCache> occ(new CirCache(sdir));
if (!occ->open(CirCache::CC_OPREAD)) {
if (reason) {
msg << "Open failed in " << sdir << " : " <<
occ->getReason() << endl;
msg << "Open failed in " << sdir << " : " << occ->getReason() << endl;
*reason = msg.str();
}
return -1;
}
// Possibly resize dest. If the dest is currently recycling, it
// will keep on. This only avoids erasing entries in dest if it is
// currently writing at EOF (growing), which will be the case if
// we are using this to compact existing file (the dest was just
// created for the purpose).
int64_t dstavail{0}, dstmaxsize{0};
bool isunique;
// Check dest size
{
std::shared_ptr<CirCache> ncc(new CirCache(ddir));
if (!ncc->open(CirCache::CC_OPREAD)) {
if (reason) {
msg << "Open failed in " << ddir << " : " << ncc->getReason() << endl;
*reason = msg.str();
}
return -1;
}
dstmaxsize = ncc->m_d->m_maxsize;
dstavail = dstmaxsize - ncc->m_d->m_nheadoffs;
isunique = ncc->m_d->m_uniquentries;
}
if (dstavail < occ->size()) {
std::shared_ptr<CirCache> ncc(new CirCache(ddir));
auto nsize = dstmaxsize + (occ->size() - dstavail) + 5*1000*1000;
LOGDEB1("CirCache::appendCC: Dstmaxsize " << displayableBytes(dstmaxsize) << " dstavail "<<
displayableBytes(dstavail) << " org size " << displayableBytes(occ->size()) <<
" nsize " << displayableBytes(nsize) << "\n");
if (!ncc->create(nsize, isunique ? CC_CRUNIQUE : CC_CRNONE)) {
if (reason) {
msg << "Open failed in " << ddir << " : " << ncc->getReason() << endl;
*reason = msg.str();
}
return -1;
}
}
// Open dest file
std::shared_ptr<CirCache> ncc(new CirCache(ddir));
if (!ncc->open(CirCache::CC_OPWRITE)) {
if (reason) {
msg << "Open failed in " << ddir << " : " <<
ncc->getReason() << endl;
msg << "Open failed in " << ddir << " : " << ncc->getReason() << endl;
*reason = msg.str();
}
return -1;
src/utils/circache.h
+21 -6
View File
@@ -67,11 +67,14 @@ public:
virtual std::string getpath();
// Set data to 0 if you just want the header
// instance == -1 means get latest. Otherwise specify from 1+
virtual bool get(const std::string& udi, std::string& dic,
std::string *data = 0, int instance = -1);
// Note: the dicp MUST have an udi entry
enum PutFlags {NoCompHint = 1};
enum PutFlags {
NoCompHint = 1, // Do not attempt compression.
};
virtual bool put(const std::string& udi, const ConfSimple *dicp,
const std::string& data, unsigned int flags = 0);
@@ -99,15 +102,27 @@ public:
/* Utility: append all entries from sdir to ddir.
*
* This does not need to be a member at all, just using the namespace here.
* ddir must already exist. It will be appropriately resized if
* needed to avoid recycling while writing the new entries.
* ** Note that if dest is not currently growing, this action
* will recycle old dest entries between the current write
* point and EOF (or up to wherever we need to write to store
* the source data) **
* Also note that if the objective is just to compact (reuse the erased
* entries space) you should first create the new circache with the
* same maxsize as the old one, else the new maxsize will be the
* current file size (current erased+active entries, with
* available space corresponding to the old erased entries).
*
* @param ddir destination circache (must be previously created
* with appropriate size)
* This method does not need to be a member at all, just using the
* namespace here.
*
* @param ddir destination circache (must exist)
* @param sdir source circache
* @ret number of entries copied or -a
*/
static int append(const std::string ddir, const std::string& sdir,
std::string *reason = 0);
static int appendCC(const std::string ddir, const std::string& sdir,
std::string *reason = 0);
protected:
CirCacheInternal *m_d;