circache: add compaction function

src/testmains/trcircache.cpp

@@ -34,6 +34,8 @@ static char usage [] =
     " -a <targetdir> <dir> [<dir> ...]: append content from existing cache(s) to target\n"
     "  The target should be first resized to hold all the data, else only\n"
     "  as many entries as capacity permit will be retained\n"
+    " -C <dir> : recover space from erased entries. May temporarily need twice the current "
+    "    space used by the cache\n";
     ;
 
 static void
@@ -54,6 +56,7 @@ static int     op_flags;
 #define OPT_u     0x100
 #define OPT_e     0x200
 #define OPT_a     0x800
+#define OPT_C     0x1000
 
 bool storeFile(CirCache& cc, const std::string fn);
 
@@ -74,24 +77,13 @@ int main(int argc, char **argv)
         }
         while (**argv)
             switch (*(*argv)++) {
-            case 'a':
+            case 'a': op_flags |= OPT_a; break;
-                op_flags |= OPT_a;
+            case 'C': op_flags |= OPT_C; break;
-                break;
+            case 'c': op_flags |= OPT_c; break;
-            case 'c':
+            case 'D': op_flags |= OPT_D; break;
-                op_flags |= OPT_c;
+            case 'd': op_flags |= OPT_d; break;
-                break;
+            case 'e': op_flags |= OPT_e; break;
-            case 'D':
+            case 'g': op_flags |= OPT_g; break;
-                op_flags |= OPT_D;
-                break;
-            case 'd':
-                op_flags |= OPT_d;
-                break;
-            case 'e':
-                op_flags |= OPT_e;
-                break;
-            case 'g':
-                op_flags |= OPT_g;
-                break;
             case 'i':
                 op_flags |= OPT_i;
                 if (argc < 2) {
@@ -102,15 +94,9 @@ int main(int argc, char **argv)
                 }
                 argc--;
                 goto b1;
-            case 'p':
+            case 'p': op_flags |= OPT_p; break;
-                op_flags |= OPT_p;
+            case 'u': op_flags |= OPT_u; break;
-                break;
+            default: Usage(); break;
-            case 'u':
-                op_flags |= OPT_u;
-                break;
-            default:
-                Usage();
-                break;
             }
 b1:
         argc--;
@@ -141,6 +127,16 @@ b1:
             cerr << "Create failed:" << cc.getReason() << endl;
             exit(1);
         }
+    } else if (op_flags & OPT_C) {
+        if (argc) {
+            Usage();
+        }
+        std::string reason;
+        if (!CirCache::compact(dir, &reason)) {
+            std::cerr << "Compact failed: " << reason << "\n";
+            return 1;
+        }
+        return 0;
     } else if (op_flags & OPT_a) {
         if (argc < 1) {
             Usage();

src/utils/circache.cpp

@@ -36,6 +36,9 @@
 #include "chrono.h"
 #include "zlibut.h"
 #include "smallut.h"
+#include "pathut.h"
+#include "wipedir.h"
+#include "copyfile.h"
 
 #ifndef _WIN32
 #include <sys/uio.h>
@@ -497,8 +500,7 @@ public:
         return CCScanHook::Continue;
     }
 
-    CCScanHook::status scan(int64_t startoffset, CCScanHook *user,
+    CCScanHook::status scan(int64_t startoffset, CCScanHook *user, bool fold = false) {
-                            bool fold = false) {
         if (m_fd < 0) {
             m_reason << "scan: not open ";
             return CCScanHook::Error;
@@ -806,6 +808,34 @@ int64_t CirCache::size()
     return st.st_size;
 }
 
+int64_t CirCache::maxsize()
+{
+    if (m_d == 0) {
+        LOGERR("CirCache::open: null data\n");
+        return -1;
+    }
+    return m_d->m_maxsize;
+}
+
+int64_t CirCache::writepos()
+{
+    if (m_d == 0) {
+        LOGERR("CirCache::open: null data\n");
+        return -1;
+    }
+    return m_d->m_nheadoffs;
+}
+
+bool CirCache::uniquentries()
+{
+    if (m_d == 0) {
+        LOGERR("CirCache::open: null data\n");
+        return -1;
+    }
+    return m_d->m_uniquentries;
+}
+
+
 class CCScanHookDump : public  CCScanHook {
 public:
     virtual status takeone(int64_t offs, const string& udi,
@@ -1337,7 +1367,7 @@ static bool copyall(std::shared_ptr<CirCache> occ,
     return true;
 }
 
-int CirCache::appendCC(const string ddir, const string& sdir, string *reason)
+int CirCache::appendCC(const string& ddir, const string& sdir, string *reason)
 {
     ostringstream msg;
     // Open source file
@@ -1406,3 +1436,110 @@ int CirCache::appendCC(const string ddir, const string& sdir, string *reason)
 
     return nentries;
 }
+
+bool CirCache::compact(const std::string& dir, std::string *reason)
+{
+    ostringstream msg;
+    // Open source file
+    std::shared_ptr<CirCache> occ(new CirCache(dir));
+    if (!occ->open(CirCache::CC_OPREAD)) {
+        msg << "CirCache::compact: open failed in " << dir << " : " << occ->getReason() << "\n";
+        LOGERR(msg.str());
+        if (reason) {
+            *reason = msg.str();
+        }
+        return -1;
+    }
+    long long avmbs;
+    if (fsocc(dir, nullptr, &avmbs) && avmbs * 1024 * 1024 < 1.2 * occ->size()) {
+        msg << "CirCache::compact: not enough space on file system";
+        LOGERR(msg.str() <<"\n");
+        if (reason) {
+            *reason = msg.str();
+        }
+        return false;
+    }
+    std::string ndir = path_cat(dir, "tmpcopy");
+    if (!path_makepath(dir, 0700)) {
+        msg << "CirCache::compact: path_makepath failed with errno " << errno;
+        LOGERR(msg.str() << "\n");
+        if (reason) {
+            *reason = msg.str();
+        }
+        return false;
+    }
+        
+    std::shared_ptr<CirCache> ncc(new CirCache(ndir));
+    if (!ncc->create(occ->size(), occ->uniquentries() ? CC_CRUNIQUE : CC_CRNONE)) {
+        msg << "CirCache::compact: Open failed in " << ndir << " : " << ncc->getReason();
+        LOGERR(msg.str() << "\n");
+        if (reason) {
+            *reason = msg.str();
+        }
+        return false;
+    }
+    int nentries;
+    if (!copyall(occ, ncc, nentries, msg)) {
+        LOGERR(msg.str() << "\n");
+        if (reason) {
+            *reason = msg.str();
+        }
+        return false;
+    }
+
+    // Close both
+    occ.reset();
+    ncc.reset();
+    // Rename new to old
+    std::string r;
+    std::string nfile = path_cat(ndir, "circache.crch").c_str();
+    std::string ofile = path_cat(dir, "circache.crch").c_str();
+    if (!renameormove(nfile.c_str(), ofile.c_str(), r)) {
+        msg << "CirCache::compact: rename: " << r;
+        LOGERR(msg.str() << "\n");
+        if (reason) {
+            *reason = msg.str();
+        }
+        return false;
+    }
+
+    // Cleanup
+    wipedir(ndir, true);
+    return true;
+}
+
+bool CirCache::burst(const std::string& ccdir, const std::string destdir, std::string *reason)
+{
+    ostringstream msg;
+    // Open source file
+    std::shared_ptr<CirCache> occ(new CirCache(ccdir));
+    if (!occ->open(CirCache::CC_OPREAD)) {
+        msg << "CirCache::burst: open failed in " << dir << " : " << occ->getReason() << "\n";
+        LOGERR(msg.str());
+        if (reason) {
+            *reason = msg.str();
+        }
+        return -1;
+    }
+    long long avmbs;
+    if (fsocc(dir, nullptr, &avmbs) && avmbs * 1024 * 1024 < 1.2 * occ->size()) {
+        msg << "CirCache::burst: not enough space on file system";
+        LOGERR(msg.str() <<"\n");
+        if (reason) {
+            *reason = msg.str();
+        }
+        return false;
+    }
+    if (!path_makepath(destdir, 0700)) {
+        msg << "CirCache::burst: path_makepath failed with errno " << errno;
+        LOGERR(msg.str() << "\n");
+        if (reason) {
+            *reason = msg.str();
+        }
+        return false;
+    }
+
+                m_d->scan(CIRCACHE_FIRSTBLOCK_SIZE, &rec, false);
+    
+}
+

src/utils/circache.h

@@ -63,6 +63,9 @@ public:
     virtual bool open(OpMode mode);
 
     virtual int64_t size();
+    virtual int64_t maxsize();
+    virtual int64_t writepos();
+    virtual bool    uniquentries();
     
     virtual std::string getpath();
 
@@ -114,16 +117,22 @@ public:
      * current file size (current erased+active entries, with
      * available space corresponding to the old erased entries).
      *
-     * This method does not need to be a member at all, just using the
-     * namespace here.
-     *
      * @param ddir destination circache (must exist)
      * @param sdir source circache
      * @ret number of entries copied or -a
      */
-    static int appendCC(const std::string ddir, const std::string& sdir,
+    static int appendCC(const std::string& ddir, const std::string& sdir,
                         std::string *reason = 0);
 
+    /* Rewrite the cache so that the space wasted to erased
+     * entries is recovered. This may need to use temporarily twice
+     * the current cache disk space.
+     */
+    static bool compact(const std::string& dir, std::string *reason = 0);
+
+    /* Extract all entries as metadata/data file pairs */
+    static bool burst(const std::string& ccdir, const std::string destdir, std::string *reason = 0);
+    
 protected:
     CirCacheInternal *m_d;
     std::string m_dir;