tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

circache: add compaction function

Browse Source
This commit is contained in:
Jean-Francois Dockes 2021-03-24 16:28:29 +01:00
parent 8fde38975a
commit efc29e11a7
3 changed files with 176 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/testmains/trcircache.cpp
View File

@ -34,6 +34,8 @@ static char usage [] =
" -a <targetdir> <dir> [<dir> ...]: append content from existing cache(s) to target\n"
" The target should be first resized to hold all the data, else only\n"
" as many entries as capacity permit will be retained\n"
" -C <dir> : recover space from erased entries. May temporarily need twice the current "
" space used by the cache\n";
;
static void
@ -54,6 +56,7 @@ static int op_flags;
#define OPT_u 0x100
#define OPT_e 0x200
#define OPT_a 0x800
#define OPT_C 0x1000
bool storeFile(CirCache& cc, const std::string fn);
@ -74,24 +77,13 @@ int main(int argc, char **argv)
}
while (**argv)
switch (*(*argv)++) {
case 'a':
op_flags |= OPT_a;
break;
case 'c':
op_flags |= OPT_c;
break;
case 'D':
op_flags |= OPT_D;
break;
case 'd':
op_flags |= OPT_d;
break;
case 'e':
op_flags |= OPT_e;
break;
case 'g':
op_flags |= OPT_g;
break;
case 'a': op_flags |= OPT_a; break;
case 'C': op_flags |= OPT_C; break;
case 'c': op_flags |= OPT_c; break;
case 'D': op_flags |= OPT_D; break;
case 'd': op_flags |= OPT_d; break;
case 'e': op_flags |= OPT_e; break;
case 'g': op_flags |= OPT_g; break;
case 'i':
op_flags |= OPT_i;
if (argc < 2) {
@ -102,15 +94,9 @@ int main(int argc, char **argv)
}
argc--;
goto b1;
case 'p':
op_flags |= OPT_p;
break;
case 'u':
op_flags |= OPT_u;
break;
default:
Usage();
break;
case 'p': op_flags |= OPT_p; break;
case 'u': op_flags |= OPT_u; break;
default: Usage(); break;
}
b1:
argc--;
@ -141,6 +127,16 @@ b1:
cerr << "Create failed:" << cc.getReason() << endl;
exit(1);
}
} else if (op_flags & OPT_C) {
if (argc) {
Usage();
}
std::string reason;
if (!CirCache::compact(dir, &reason)) {
std::cerr << "Compact failed: " << reason << "\n";
return 1;
}
return 0;
} else if (op_flags & OPT_a) {
if (argc < 1) {
Usage();

143
src/utils/circache.cpp
View File

@ -36,6 +36,9 @@
#include "chrono.h"
#include "zlibut.h"
#include "smallut.h"
#include "pathut.h"
#include "wipedir.h"
#include "copyfile.h"
#ifndef _WIN32
#include <sys/uio.h>
@ -497,8 +500,7 @@ public:
return CCScanHook::Continue;
}
CCScanHook::status scan(int64_t startoffset, CCScanHook *user,
bool fold = false) {
CCScanHook::status scan(int64_t startoffset, CCScanHook *user, bool fold = false) {
if (m_fd < 0) {
m_reason << "scan: not open ";
return CCScanHook::Error;
@ -806,6 +808,34 @@ int64_t CirCache::size()
return st.st_size;
}
int64_t CirCache::maxsize()
{
if (m_d == 0) {
LOGERR("CirCache::open: null data\n");
return -1;
}
return m_d->m_maxsize;
}
int64_t CirCache::writepos()
{
if (m_d == 0) {
LOGERR("CirCache::open: null data\n");
return -1;
}
return m_d->m_nheadoffs;
}
bool CirCache::uniquentries()
{
if (m_d == 0) {
LOGERR("CirCache::open: null data\n");
return -1;
}
return m_d->m_uniquentries;
}
class CCScanHookDump : public CCScanHook {
public:
virtual status takeone(int64_t offs, const string& udi,
@ -1337,7 +1367,7 @@ static bool copyall(std::shared_ptr<CirCache> occ,
return true;
}
int CirCache::appendCC(const string ddir, const string& sdir, string *reason)
int CirCache::appendCC(const string& ddir, const string& sdir, string *reason)
{
ostringstream msg;
// Open source file
@ -1406,3 +1436,110 @@ int CirCache::appendCC(const string ddir, const string& sdir, string *reason)
return nentries;
}
bool CirCache::compact(const std::string& dir, std::string *reason)
{
ostringstream msg;
// Open source file
std::shared_ptr<CirCache> occ(new CirCache(dir));
if (!occ->open(CirCache::CC_OPREAD)) {
msg << "CirCache::compact: open failed in " << dir << " : " << occ->getReason() << "\n";
LOGERR(msg.str());
if (reason) {
*reason = msg.str();
}
return -1;
}
long long avmbs;
if (fsocc(dir, nullptr, &avmbs) && avmbs * 1024 * 1024 < 1.2 * occ->size()) {
msg << "CirCache::compact: not enough space on file system";
LOGERR(msg.str() <<"\n");
if (reason) {
*reason = msg.str();
}
return false;
}
std::string ndir = path_cat(dir, "tmpcopy");
if (!path_makepath(dir, 0700)) {
msg << "CirCache::compact: path_makepath failed with errno " << errno;
LOGERR(msg.str() << "\n");
if (reason) {
*reason = msg.str();
}
return false;
}
std::shared_ptr<CirCache> ncc(new CirCache(ndir));
if (!ncc->create(occ->size(), occ->uniquentries() ? CC_CRUNIQUE : CC_CRNONE)) {
msg << "CirCache::compact: Open failed in " << ndir << " : " << ncc->getReason();
LOGERR(msg.str() << "\n");
if (reason) {
*reason = msg.str();
}
return false;
}
int nentries;
if (!copyall(occ, ncc, nentries, msg)) {
LOGERR(msg.str() << "\n");
if (reason) {
*reason = msg.str();
}
return false;
}
// Close both
occ.reset();
ncc.reset();
// Rename new to old
std::string r;
std::string nfile = path_cat(ndir, "circache.crch").c_str();
std::string ofile = path_cat(dir, "circache.crch").c_str();
if (!renameormove(nfile.c_str(), ofile.c_str(), r)) {
msg << "CirCache::compact: rename: " << r;
LOGERR(msg.str() << "\n");
if (reason) {
*reason = msg.str();
}
return false;
}
// Cleanup
wipedir(ndir, true);
return true;
}
bool CirCache::burst(const std::string& ccdir, const std::string destdir, std::string *reason)
{
ostringstream msg;
// Open source file
std::shared_ptr<CirCache> occ(new CirCache(ccdir));
if (!occ->open(CirCache::CC_OPREAD)) {
msg << "CirCache::burst: open failed in " << dir << " : " << occ->getReason() << "\n";
LOGERR(msg.str());
if (reason) {
*reason = msg.str();
}
return -1;
}
long long avmbs;
if (fsocc(dir, nullptr, &avmbs) && avmbs * 1024 * 1024 < 1.2 * occ->size()) {
msg << "CirCache::burst: not enough space on file system";
LOGERR(msg.str() <<"\n");
if (reason) {
*reason = msg.str();
}
return false;
}
if (!path_makepath(destdir, 0700)) {
msg << "CirCache::burst: path_makepath failed with errno " << errno;
LOGERR(msg.str() << "\n");
if (reason) {
*reason = msg.str();
}
return false;
}
m_d->scan(CIRCACHE_FIRSTBLOCK_SIZE, &rec, false);
}

17
src/utils/circache.h
View File

@ -63,6 +63,9 @@ public:
virtual bool open(OpMode mode);
virtual int64_t size();
virtual int64_t maxsize();
virtual int64_t writepos();
virtual bool uniquentries();
virtual std::string getpath();
@ -114,16 +117,22 @@ public:
* current file size (current erased+active entries, with
* available space corresponding to the old erased entries).
*
* This method does not need to be a member at all, just using the
* namespace here.
*
* @param ddir destination circache (must exist)
* @param sdir source circache
* @ret number of entries copied or -a
*/
static int appendCC(const std::string ddir, const std::string& sdir,
static int appendCC(const std::string& ddir, const std::string& sdir,
std::string *reason = 0);
/* Rewrite the cache so that the space wasted to erased
* entries is recovered. This may need to use temporarily twice
* the current cache disk space.
*/
static bool compact(const std::string& dir, std::string *reason = 0);
/* Extract all entries as metadata/data file pairs */
static bool burst(const std::string& ccdir, const std::string destdir, std::string *reason = 0);
protected:
CirCacheInternal *m_d;
std::string m_dir;