diff --git a/src/testmains/trcircache.cpp b/src/testmains/trcircache.cpp index 22477eb3..73f0e945 100644 --- a/src/testmains/trcircache.cpp +++ b/src/testmains/trcircache.cpp @@ -34,6 +34,8 @@ static char usage [] = " -a [ ...]: append content from existing cache(s) to target\n" " The target should be first resized to hold all the data, else only\n" " as many entries as capacity permit will be retained\n" + " -C : recover space from erased entries. May temporarily need twice the current " + " space used by the cache\n"; ; static void @@ -54,6 +56,7 @@ static int op_flags; #define OPT_u 0x100 #define OPT_e 0x200 #define OPT_a 0x800 +#define OPT_C 0x1000 bool storeFile(CirCache& cc, const std::string fn); @@ -74,24 +77,13 @@ int main(int argc, char **argv) } while (**argv) switch (*(*argv)++) { - case 'a': - op_flags |= OPT_a; - break; - case 'c': - op_flags |= OPT_c; - break; - case 'D': - op_flags |= OPT_D; - break; - case 'd': - op_flags |= OPT_d; - break; - case 'e': - op_flags |= OPT_e; - break; - case 'g': - op_flags |= OPT_g; - break; + case 'a': op_flags |= OPT_a; break; + case 'C': op_flags |= OPT_C; break; + case 'c': op_flags |= OPT_c; break; + case 'D': op_flags |= OPT_D; break; + case 'd': op_flags |= OPT_d; break; + case 'e': op_flags |= OPT_e; break; + case 'g': op_flags |= OPT_g; break; case 'i': op_flags |= OPT_i; if (argc < 2) { @@ -102,15 +94,9 @@ int main(int argc, char **argv) } argc--; goto b1; - case 'p': - op_flags |= OPT_p; - break; - case 'u': - op_flags |= OPT_u; - break; - default: - Usage(); - break; + case 'p': op_flags |= OPT_p; break; + case 'u': op_flags |= OPT_u; break; + default: Usage(); break; } b1: argc--; @@ -141,6 +127,16 @@ b1: cerr << "Create failed:" << cc.getReason() << endl; exit(1); } + } else if (op_flags & OPT_C) { + if (argc) { + Usage(); + } + std::string reason; + if (!CirCache::compact(dir, &reason)) { + std::cerr << "Compact failed: " << reason << "\n"; + return 1; + } + return 0; } else if (op_flags & OPT_a) { if (argc < 1) { Usage(); diff --git a/src/utils/circache.cpp b/src/utils/circache.cpp index 85ba0282..dabe7ea3 100644 --- a/src/utils/circache.cpp +++ b/src/utils/circache.cpp @@ -36,6 +36,9 @@ #include "chrono.h" #include "zlibut.h" #include "smallut.h" +#include "pathut.h" +#include "wipedir.h" +#include "copyfile.h" #ifndef _WIN32 #include @@ -497,8 +500,7 @@ public: return CCScanHook::Continue; } - CCScanHook::status scan(int64_t startoffset, CCScanHook *user, - bool fold = false) { + CCScanHook::status scan(int64_t startoffset, CCScanHook *user, bool fold = false) { if (m_fd < 0) { m_reason << "scan: not open "; return CCScanHook::Error; @@ -806,6 +808,34 @@ int64_t CirCache::size() return st.st_size; } +int64_t CirCache::maxsize() +{ + if (m_d == 0) { + LOGERR("CirCache::open: null data\n"); + return -1; + } + return m_d->m_maxsize; +} + +int64_t CirCache::writepos() +{ + if (m_d == 0) { + LOGERR("CirCache::open: null data\n"); + return -1; + } + return m_d->m_nheadoffs; +} + +bool CirCache::uniquentries() +{ + if (m_d == 0) { + LOGERR("CirCache::open: null data\n"); + return -1; + } + return m_d->m_uniquentries; +} + + class CCScanHookDump : public CCScanHook { public: virtual status takeone(int64_t offs, const string& udi, @@ -1337,7 +1367,7 @@ static bool copyall(std::shared_ptr occ, return true; } -int CirCache::appendCC(const string ddir, const string& sdir, string *reason) +int CirCache::appendCC(const string& ddir, const string& sdir, string *reason) { ostringstream msg; // Open source file @@ -1406,3 +1436,110 @@ int CirCache::appendCC(const string ddir, const string& sdir, string *reason) return nentries; } + +bool CirCache::compact(const std::string& dir, std::string *reason) +{ + ostringstream msg; + // Open source file + std::shared_ptr occ(new CirCache(dir)); + if (!occ->open(CirCache::CC_OPREAD)) { + msg << "CirCache::compact: open failed in " << dir << " : " << occ->getReason() << "\n"; + LOGERR(msg.str()); + if (reason) { + *reason = msg.str(); + } + return -1; + } + long long avmbs; + if (fsocc(dir, nullptr, &avmbs) && avmbs * 1024 * 1024 < 1.2 * occ->size()) { + msg << "CirCache::compact: not enough space on file system"; + LOGERR(msg.str() <<"\n"); + if (reason) { + *reason = msg.str(); + } + return false; + } + std::string ndir = path_cat(dir, "tmpcopy"); + if (!path_makepath(dir, 0700)) { + msg << "CirCache::compact: path_makepath failed with errno " << errno; + LOGERR(msg.str() << "\n"); + if (reason) { + *reason = msg.str(); + } + return false; + } + + std::shared_ptr ncc(new CirCache(ndir)); + if (!ncc->create(occ->size(), occ->uniquentries() ? CC_CRUNIQUE : CC_CRNONE)) { + msg << "CirCache::compact: Open failed in " << ndir << " : " << ncc->getReason(); + LOGERR(msg.str() << "\n"); + if (reason) { + *reason = msg.str(); + } + return false; + } + int nentries; + if (!copyall(occ, ncc, nentries, msg)) { + LOGERR(msg.str() << "\n"); + if (reason) { + *reason = msg.str(); + } + return false; + } + + // Close both + occ.reset(); + ncc.reset(); + // Rename new to old + std::string r; + std::string nfile = path_cat(ndir, "circache.crch").c_str(); + std::string ofile = path_cat(dir, "circache.crch").c_str(); + if (!renameormove(nfile.c_str(), ofile.c_str(), r)) { + msg << "CirCache::compact: rename: " << r; + LOGERR(msg.str() << "\n"); + if (reason) { + *reason = msg.str(); + } + return false; + } + + // Cleanup + wipedir(ndir, true); + return true; +} + +bool CirCache::burst(const std::string& ccdir, const std::string destdir, std::string *reason) +{ + ostringstream msg; + // Open source file + std::shared_ptr occ(new CirCache(ccdir)); + if (!occ->open(CirCache::CC_OPREAD)) { + msg << "CirCache::burst: open failed in " << dir << " : " << occ->getReason() << "\n"; + LOGERR(msg.str()); + if (reason) { + *reason = msg.str(); + } + return -1; + } + long long avmbs; + if (fsocc(dir, nullptr, &avmbs) && avmbs * 1024 * 1024 < 1.2 * occ->size()) { + msg << "CirCache::burst: not enough space on file system"; + LOGERR(msg.str() <<"\n"); + if (reason) { + *reason = msg.str(); + } + return false; + } + if (!path_makepath(destdir, 0700)) { + msg << "CirCache::burst: path_makepath failed with errno " << errno; + LOGERR(msg.str() << "\n"); + if (reason) { + *reason = msg.str(); + } + return false; + } + + m_d->scan(CIRCACHE_FIRSTBLOCK_SIZE, &rec, false); + +} + diff --git a/src/utils/circache.h b/src/utils/circache.h index e3667a54..1fd62e93 100644 --- a/src/utils/circache.h +++ b/src/utils/circache.h @@ -63,6 +63,9 @@ public: virtual bool open(OpMode mode); virtual int64_t size(); + virtual int64_t maxsize(); + virtual int64_t writepos(); + virtual bool uniquentries(); virtual std::string getpath(); @@ -114,16 +117,22 @@ public: * current file size (current erased+active entries, with * available space corresponding to the old erased entries). * - * This method does not need to be a member at all, just using the - * namespace here. - * * @param ddir destination circache (must exist) * @param sdir source circache * @ret number of entries copied or -a */ - static int appendCC(const std::string ddir, const std::string& sdir, + static int appendCC(const std::string& ddir, const std::string& sdir, std::string *reason = 0); + /* Rewrite the cache so that the space wasted to erased + * entries is recovered. This may need to use temporarily twice + * the current cache disk space. + */ + static bool compact(const std::string& dir, std::string *reason = 0); + + /* Extract all entries as metadata/data file pairs */ + static bool burst(const std::string& ccdir, const std::string destdir, std::string *reason = 0); + protected: CirCacheInternal *m_d; std::string m_dir;