tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

1st proto for GUI webcache editor

Browse Source
This commit is contained in:
Jean-Francois Dockes
2016-04-13 15:52:41 +02:00
parent 5d6b1ea2a1
commit 241586a3a7
11 changed files with 465 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/circache.cpp
+112 -177
View File
@@ -30,6 +30,8 @@
#include <zlib.h>
#include "chrono.h"
#include MEMORY_INCLUDE
#ifndef _WIN32
#include <sys/uio.h>
@@ -857,7 +859,7 @@ public:
};
// instance == -1 means get latest. Otherwise specify from 1+
bool CirCache::get(const string& udi, string& dic, string& data, int instance)
bool CirCache::get(const string& udi, string& dic, string *data, int instance)
{
Chrono chron;
if (m_d->m_fd < 0) {
@@ -900,7 +902,7 @@ bool CirCache::get(const string& udi, string& dic, string& data, int instance)
}
// Did we read an appropriate entry ?
if (o_good != 0 && (instance == -1 || instance == finst)) {
bool ret = m_d->readDicData(o_good, d_good, dic, &data);
bool ret = m_d->readDicData(o_good, d_good, dic, data);
LOGDEB0(("Circache::get: hfound, %d mS\n",
chron.millis()));
return ret;
@@ -920,8 +922,7 @@ bool CirCache::get(const string& udi, string& dic, string& data, int instance)
} else if (ret != CCScanHook::Stop) {
return false;
}
bool bret =
m_d->readDicData(getter.m_offs, getter.m_hd, dic, &data);
bool bret = m_d->readDicData(getter.m_offs, getter.m_hd, dic, data);
LOGDEB0(("Circache::get: scanfound, %d mS\n", chron.millis()));
return bret;
@@ -943,8 +944,8 @@ bool CirCache::erase(const string& udi)
// If the mem cache is not up to date, update it, we're too lazy
// to do a scan
if (!m_d->m_ofskhcplt) {
string dic, data;
get("nosuchudi probably exists", dic, data);
string dic;
get("nosuchudi probably exists", dic);
if (!m_d->m_ofskhcplt) {
LOGERR(("CirCache::erase : cache not updated after get\n"));
return false;
@@ -1265,13 +1266,13 @@ bool CirCache::getCurrentUdi(string& udi)
return true;
}
bool CirCache::getCurrent(string& udi, string& dic, string& data)
bool CirCache::getCurrent(string& udi, string& dic, string *data)
{
if (m_d == 0) {
LOGERR(("CirCache::getCurrent: null data\n"));
return false;
}
if (!m_d->readDicData(m_d->m_itoffs, m_d->m_ithd, dic, &data)) {
if (!m_d->readDicData(m_d->m_itoffs, m_d->m_ithd, dic, data)) {
return false;
}
@@ -1367,6 +1368,85 @@ static bool inflateToDynBuf(void* inp, UINT inlen, void **outpp, UINT *outlenp)
return true;
}
// Copy all entries from occ to ncc. Both are already open.
static bool copyall(STD_SHARED_PTR<CirCache> occ,
STD_SHARED_PTR<CirCache> ncc, int& nentries,
ostringstream& msg)
{
bool eof = false;
if (!occ->rewind(eof)) {
if (!eof) {
msg << "Initial rewind failed" << endl;
return false;
}
}
nentries = 0;
while (!eof) {
string udi, sdic, data;
if (!occ->getCurrent(udi, sdic, &data)) {
msg << "getCurrent failed: " << occ->getReason() << endl;
return false;
}
// Shouldn't getcurrent deal with this ?
if (sdic.size() == 0) {
//cerr << "Skip empty entry" << endl;
occ->next(eof);
continue;
}
ConfSimple dic(sdic);
if (!dic.ok()) {
msg << "Could not parse entry attributes dic" << endl;
return false;
}
//cerr << "UDI: " << udi << endl;
if (!ncc->put(udi, &dic, data)) {
msg << "put failed: " << ncc->getReason() << " sdic [" << sdic <<
"]" << endl;
return false;
}
nentries++;
occ->next(eof);
}
return true;
}
// Append all entries from sdir to ddir
int CirCache::append(const string ddir, const string& sdir, string *reason)
{
ostringstream msg;
// Open source file
STD_SHARED_PTR<CirCache> occ(new CirCache(sdir));
if (!occ->open(CirCache::CC_OPREAD)) {
if (reason) {
msg << "Open failed in " << sdir << " : " <<
occ->getReason() << endl;
*reason = msg.str();
}
return -1;
}
// Open dest file
STD_SHARED_PTR<CirCache> ncc(new CirCache(ddir));
if (!ncc->open(CirCache::CC_OPWRITE)) {
if (reason) {
msg << "Open failed in " << ddir << " : " <<
ncc->getReason() << endl;
*reason = msg.str();
}
return -1;
}
int nentries;
if (!copyall(occ, ncc, nentries, msg)) {
if (reason) {
*reason = msg.str();
}
return -1;
}
return nentries;
}
#else // TEST ->
#include "autoconfig.h"
@@ -1380,163 +1460,17 @@ static bool inflateToDynBuf(void* inp, UINT inlen, void **outpp, UINT *outlenp)
#include <string>
#include <iostream>
#include MEMORY_INCLUDE
#include "circache.h"
#include "fileudi.h"
#include "conftree.h"
#include "readfile.h"
#include "debuglog.h"
#include "smallut.h"
using namespace std;
// Copy all entries from occ to ncc. Both are already open.
bool copyall(STD_SHARED_PTR<CirCache> occ,
STD_SHARED_PTR<CirCache> ncc, int& nentries)
{
bool eof = false;
if (!occ->rewind(eof)) {
if (!eof) {
cerr << "Initial rewind failed" << endl;
return false;
}
}
nentries = 0;
while (!eof) {
string udi, sdic, data;
if (!occ->getCurrent(udi, sdic, data)) {
cerr << "getCurrent failed: " << occ->getReason() << endl;
return false;
}
// Shouldn't getcurrent deal with this ?
if (sdic.size() == 0) {
//cerr << "Skip empty entry" << endl;
occ->next(eof);
continue;
}
ConfSimple dic(sdic);
if (!dic.ok()) {
cerr << "Could not parse entry attributes dic" << endl;
return false;
}
//cerr << "UDI: " << udi << endl;
if (!ncc->put(udi, &dic, data)) {
cerr << "put failed: " << ncc->getReason() << " sdic [" << sdic <<
"]" << endl;
return false;
}
nentries++;
occ->next(eof);
}
return true;
}
#warning resizecc is not useful, create(newsize) works !
// Wrote the following at a point where I thought that simple resizing
// did not work. Upon further study (or updates?), it appears it does
// ! So the following code is not useful actually
// Resize circache. This can't be done easily if the write point is
// inside the file (we already reached the old max size). We create a
// new file with the new size and copy the old entries into it. The
// old file is then renamed into a backup and the new file renamed in
// place.
bool resizecc(const string& dir, int newmbs)
{
// Create object for existing file and get the file name
STD_SHARED_PTR<CirCache> occ(new CirCache(dir));
string ofn = occ->getpath();
// Check for previous backup
string backupfn = ofn + ".orig";
if (access(backupfn.c_str(), 0) >= 0) {
cerr << "Backup file " << backupfn <<
" exists, please move it out of the way" << endl;
return false;
}
if (!occ->open(CirCache::CC_OPREAD)) {
cerr << "Open failed in " << dir << " : " << occ->getReason() << endl;
return false;
}
// Create the new empty file in a temporary directory
string tmpdir = path_cat(dir, "tmp");
if (access(tmpdir.c_str(), 0) < 0) {
if (mkdir(tmpdir.c_str(), 0700) < 0) {
cerr << "Cant create temporary directory " << tmpdir << " ";
perror("mkdir");
return false;
}
}
STD_SHARED_PTR<CirCache> ncc(new CirCache(tmpdir));
string nfn = ncc->getpath();
if (!ncc->create(off_t(newmbs) * 1000 * 1024,
CirCache::CC_CRUNIQUE | CirCache::CC_CRTRUNCATE)) {
cerr << "Cant create new file in " << tmpdir << " : " <<
ncc->getReason() << endl;
return false;
}
int nentries;
if (!copyall(occ, ncc, nentries)) {
cerr << "Copy failed\n";
return false;
}
// Done with our objects here, there is no close() method, so
// delete them
occ.reset();
ncc.reset();
// Create backup by renaming the old file
if (rename(ofn.c_str(), backupfn.c_str()) < 0) {
cerr << "Could not create backup " << backupfn << " : ";
perror("rename");
return false;
}
cout << "Created backup file " << backupfn << endl;
// Move the new file in place.
if (rename(nfn.c_str(), ofn.c_str()) < 0) {
cerr << "Could not rename new file from " << nfn << " to " <<
ofn << " : ";
perror("rename");
return false;
}
cout << "Resize done, copied " << nentries << " entries " << endl;
return true;
}
// Append all entries from sdir to ddir
bool appendcc(const string ddir, const string& sdir)
{
// Open source file
STD_SHARED_PTR<CirCache> occ(new CirCache(sdir));
if (!occ->open(CirCache::CC_OPREAD)) {
cerr << "Open failed in " << sdir << " : " << occ->getReason() << endl;
return false;
}
// Open dest file
STD_SHARED_PTR<CirCache> ncc(new CirCache(ddir));
if (!ncc->open(CirCache::CC_OPWRITE)) {
cerr << "Open failed in " << ddir << " : " << ncc->getReason() << endl;
return false;
}
int nentries;
if (!copyall(occ, ncc, nentries)) {
cerr << "Copy failed\n";
return false;
}
occ.reset();
ncc.reset();
cout << "Copy done, copied " << nentries << " entries " << endl;
return true;
}
static char *thisprog;
static char usage [] =
@@ -1546,7 +1480,6 @@ static char usage [] =
" -g [-i instance] [-D] <dirname> <udi>: get\n"
" -D: also dump data\n"
" -e <dirname> <udi> : erase\n"
" -s <dirname> <newmbs> : resize\n"
" -a <targetdir> <dir> [<dir> ...]: append old content to target\n"
" The target should be first resized to hold all the data, else only\n"
" as many entries as capacity permit will be retained\n"
@@ -1569,7 +1502,6 @@ static int op_flags;
#define OPT_D 0x80
#define OPT_u 0x100
#define OPT_e 0x200
#define OPT_s 0x400
#define OPT_a 0x800
int main(int argc, char **argv)
@@ -1620,9 +1552,6 @@ int main(int argc, char **argv)
case 'p':
op_flags |= OPT_p;
break;
case 's':
op_flags |= OPT_s;
break;
case 'u':
op_flags |= OPT_u;
break;
@@ -1660,21 +1589,14 @@ b1:
cerr << "Create failed:" << cc.getReason() << endl;
exit(1);
}
} else if (op_flags & OPT_s) {
if (argc != 1) {
Usage();
}
int newmbs = atoi(*argv++);
argc--;
if (!resizecc(dir, newmbs)) {
exit(1);
}
} else if (op_flags & OPT_a) {
if (argc < 1) {
Usage();
}
while (argc) {
if (!appendcc(dir, *argv++)) {
string reason;
if (CirCache::append(dir, *argv++, &reason) < 0) {
cerr << reason << endl;
return 1;
}
argc--;
@@ -1698,11 +1620,24 @@ b1:
}
string udi;
make_udi(fn, "", udi);
sprintf(dic, "#whatever...\nmimetype = text/plain\nudi=%s\n",
udi.c_str());
string sdic;
sdic.assign(dic, strlen(dic));
ConfSimple conf(sdic);
string cmd("xdg-mime query filetype ");
// Should do more quoting here...
cmd += "'" + fn + "'";
FILE *fp = popen(cmd.c_str(), "r");
char* buf=0;
size_t sz = 0;
::getline(&buf, &sz, fp);
pclose(fp);
string mimetype(buf);
free(buf);
trimstring(mimetype, "\n\r");
cout << "Got [" << mimetype << "]\n";
string s;
ConfSimple conf(s);
conf.set("udi", udi);
conf.set("mimetype", mimetype);
//ostringstream str; conf.write(str); cout << str.str() << endl;
if (!cc.put(udi, &conf, data, 0)) {
cerr << "Put failed: " << cc.getReason() << endl;
@@ -1722,7 +1657,7 @@ b1:
string udi = *argv++;
argc--;
string dic, data;
if (!cc.get(udi, dic, data, instance)) {
if (!cc.get(udi, dic, &data, instance)) {
cerr << "Get failed: " << cc.getReason() << endl;
exit(1);
}
src/utils/circache.h
+15 -2
View File
@@ -64,8 +64,9 @@ public:
virtual std::string getpath();
// Set data to 0 if you just want the header
virtual bool get(const std::string& udi, std::string& dic,
std::string& data, int instance = -1);
std::string *data = 0, int instance = -1);
// Note: the dicp MUST have an udi entry
enum PutFlags {NoCompHint = 1};
@@ -84,7 +85,7 @@ public:
virtual bool rewind(bool& eof);
/** Get entry under cursor */
virtual bool getCurrent(std::string& udi, std::string& dic,
std::string& data);
std::string *data = 0);
/** Get current entry udi only. Udi can be empty (erased empty), caller
* should call again */
virtual bool getCurrentUdi(std::string& udi);
@@ -94,6 +95,18 @@ public:
/* Debug. This writes the entry headers to stdout */
virtual bool dump();
/* Utility: append all entries from sdir to ddir.
*
* This does not need to be a member at all, just using the namespace here.
*
* @param ddir destination circache (must be previously created
* with appropriate size)
* @param sdir source circache
* @ret number of entries copied or -a
*/
static int append(const std::string ddir, const std::string& sdir,
std::string *reason = 0);
protected:
CirCacheInternal *m_d;
std::string m_dir;