tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix format string used to generate/scan circache headers.

Browse Source 
Use _ not . as prefix for webqueue metadata files
Fix log messages and indent
This commit is contained in:
Jean-Francois Dockes 2019-11-24 15:02:30 +01:00
parent 726781434e
commit a1122c4e8a
3 changed files with 204 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/filters/recoll-we-move-files.py
View File

@ -145,6 +145,6 @@ for hash in cfiles.keys():
shutil.move(os.path.join(downloadsdir, cfiles[hash]),
os.path.join(webqueuedir, newname))
shutil.move(os.path.join(downloadsdir, mfiles[hash]),
os.path.join(webqueuedir, "." + newname))
os.path.join(webqueuedir, "_" + newname))

27
src/index/webqueue.cpp
View File

@ -74,7 +74,7 @@ public:
ll--;
}
line.assign(cline, ll);
LOGDEB2("WebQueueDotFile:readLine: [" << (line) << "]\n" );
LOGDEB2("WebQueueDotFile:readLine: [" << line << "]\n");
return true;
}
@ -85,7 +85,7 @@ public:
m_input.open(m_fn.c_str(), ios::in);
if (!m_input.good()) {
LOGERR("WebQueueDotFile: open failed for [" << (m_fn) << "]\n" );
LOGERR("WebQueueDotFile: open failed for [" << m_fn << "]\n");
return false;
}
@ -258,10 +258,10 @@ bool WebQueueIndexer::index()
{
if (!m_db)
return false;
LOGDEB("WebQueueIndexer::processqueue: [" << (m_queuedir) << "]\n" );
LOGDEB("WebQueueIndexer::processqueue: [" << m_queuedir << "]\n");
m_config->setKeyDir(m_queuedir);
if (!path_makepath(m_queuedir, 0700)) {
LOGERR("WebQueueIndexer:: can't create queuedir [" << (m_queuedir) << "] errno " << (errno) << "\n" );
LOGERR("WebQueueIndexer:: can't create queuedir [" << m_queuedir << "] errno " << errno << "\n");
return false;
}
if (!m_cache || !m_cache->cc()) {
@ -308,7 +308,7 @@ bool WebQueueIndexer::index()
FsTreeWalker walker(FsTreeWalker::FtwNoRecurse);
walker.addSkippedName(".*");
FsTreeWalker::Status status = walker.walk(m_queuedir, *this);
LOGDEB("WebQueueIndexer::processqueue: done: status " << (status) << "\n" );
LOGDEB("WebQueueIndexer::processqueue: done: status " << status << "\n");
return true;
}
@ -374,8 +374,8 @@ WebQueueIndexer::processone(const string &path,
return FsTreeWalker::FtwOk;
string dotpath = path_cat(path_getfather(path),
string(".") + path_getsimple(path));
LOGDEB("WebQueueIndexer: prc1: [" << (path) << "]\n" );
string("_") + path_getsimple(path));
LOGDEB("WebQueueIndexer: prc1: [" << path << "]\n");
WebQueueDotFile dotfile(m_config, dotpath);
Rcl::Doc dotdoc;
@ -389,7 +389,7 @@ WebQueueIndexer::processone(const string &path,
udipath = path_cat(dotdoc.meta[Rcl::Doc::keybght], url_gpath(dotdoc.url));
make_udi(udipath, cstr_null, udi);
LOGDEB("WebQueueIndexer: prc1: udi [" << (udi) << "]\n" );
LOGDEB("WebQueueIndexer: prc1: udi [" << udi << "]\n");
char ascdate[30];
sprintf(ascdate, "%ld", long(stp->st_mtime));
@ -462,7 +462,7 @@ WebQueueIndexer::processone(const string &path,
goto out;
}
if (!m_cache->cc()->put(udi, &dotfile.m_fields, fdata, 0)) {
LOGERR("WebQueueIndexer::prc1: cache_put failed; " << (m_cache->cc()->getReason()) << "\n" );
LOGERR("WebQueueIndexer::prc1: cache_put failed; " << m_cache->cc()->getReason() << "\n");
goto out;
}
}
@ -470,9 +470,12 @@ WebQueueIndexer::processone(const string &path,
dounlink = true;
out:
if (dounlink) {
unlink(path.c_str());
unlink(dotpath.c_str());
if (unlink(path.c_str())) {
LOGSYSERR("WebQueueIndexer::processone", "unlink", path);
}
if (unlink(dotpath.c_str())) {
LOGSYSERR("WebQueueIndexer::processone", "unlink", dotpath);
}
}
return FsTreeWalker::FtwOk;
}

84
src/utils/circache.cpp
View File

@ -29,6 +29,7 @@
#include "safeunistd.h"
#include <assert.h>
#include <memory.h>
#include <inttypes.h>
#include <memory>
@ -72,9 +73,6 @@ static ssize_t writev(int fd, const struct iovec *iov, int iovcnt)
#include "md5.h"
using namespace std;
typedef unsigned char UCHAR;
typedef unsigned int UINT;
typedef unsigned long ULONG;
/** Temp buffer with automatic deallocation */
struct TempBuf {
@ -125,17 +123,17 @@ struct TempBuf {
#define CIRCACHE_FIRSTBLOCK_SIZE 1024
// Entry header.
// 3 x 32 bits sizes as hex integers + 1 x 16 bits flag + at least 1 zero
// 15 + 3x(9) + 3 + 1 = 46
static const char *headerformat = "circacheSizes = %x %x %x %hx";
// 2x32 1x64 bits ints as hex integers + 1 x 16 bits flag + at least 1 zero
// 15 + 2x9 + 17 + 3 + 1 = 54
static const char *headerformat = "circacheSizes = %x %x %llx %hx";
#define CIRCACHE_HEADER_SIZE 64
class EntryHeaderData {
public:
EntryHeaderData() : dicsize(0), datasize(0), padsize(0), flags(0) {}
UINT dicsize;
UINT datasize;
UINT padsize;
unsigned int dicsize;
unsigned int datasize;
uint64_t padsize;
unsigned short flags;
};
enum EntryFlags {EFNone = 0, EFDataCompressed = 1};
@ -158,13 +156,13 @@ public:
#define UDIHLEN 4
class UdiH {
public:
UCHAR h[UDIHLEN];
unsigned char h[UDIHLEN];
UdiH(const string& udi) {
MD5_CTX ctx;
MD5Init(&ctx);
MD5Update(&ctx, (const UCHAR*)udi.c_str(), udi.length());
UCHAR md[16];
MD5Update(&ctx, (const unsigned char*)udi.c_str(), udi.length());
unsigned char md[16];
MD5Final(md, &ctx);
memcpy(h, md, UDIHLEN);
}
@ -237,13 +235,13 @@ public:
bool khEnter(const string& udi, int64_t ofs) {
UdiH h(udi);
LOGDEB2("Circache::khEnter: h " << (h.asHexString()) << " offs " << ((ULONG)ofs) << " udi [" << (udi) << "]\n" );
LOGDEB2("Circache::khEnter: h " << h.asHexString() << " offs " << ofs << " udi [" << udi << "]\n");
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
if (p.first != m_ofskh.end() && p.first->first == h) {
for (kh_type::iterator it = p.first; it != p.second; it++) {
LOGDEB2("Circache::khEnter: col h " << (it->first.asHexString()) << ", ofs " << ((ULONG)it->second) << "\n" );
LOGDEB2("Circache::khEnter: col h " << it->first.asHexString() << ", ofs " << it->second << "\n");
if (it->second == ofs) {
// (h,offs) already there. Happens
LOGDEB2("Circache::khEnter: already there\n");
@ -258,7 +256,7 @@ public:
void khDump() {
for (kh_type::const_iterator it = m_ofskh.begin();
it != m_ofskh.end(); it++) {
LOGDEB("Circache::KHDUMP: " << (it->first.asHexString()) << " " << ((ULONG)it->second) << "\n" );
LOGDEB("Circache::KHDUMP: " << it->first.asHexString() << " " << it->second << "\n");
}
}
@ -270,7 +268,7 @@ public:
UdiH h(udi);
LOGDEB2("Circache::khFind: h " << (h.asHexString()) << " udi [" << (udi) << "]\n" );
LOGDEB2("Circache::khFind: h " << h.asHexString() << " udi [" << udi << "]\n");
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
@ -282,7 +280,7 @@ public:
LOGDEB("KHFIND: SECOND END()\n");
}
if (!(p.first->first == h))
LOGDEB("KHFIND: NOKEY: " << (p.first->first.asHexString()) << " " << (p.second->first.asHexString()) << "\n" );
LOGDEB("KHFIND: NOKEY: " << p.first->first.asHexString() << " " << p.second->first.asHexString() << "\n");
#endif
if (p.first == m_ofskh.end() || !(p.first->first == h)) {
@ -496,7 +494,8 @@ public:
offset << " [" << bf << "]";
return CCScanHook::Error;
}
LOGDEB2("Circache:readEntryHeader: dcsz " << (d.dicsize) << " dtsz " << (d.datasize) << " pdsz " << (d.padsize) << " flgs " << (d.flags) << "\n" );
LOGDEB2("Circache:readEntryHeader: dcsz " << d.dicsize << " dtsz " << d.datasize << " pdsz " << d.padsize <<
" flgs " << d.flags << "\n");
return CCScanHook::Continue;
}
@ -653,7 +652,7 @@ CirCache::CirCache(const string& dir)
: m_dir(dir)
{
m_d = new CirCacheInternal;
LOGDEB0("CirCache: [" << (m_dir) << "]\n" );
LOGDEB0("CirCache: [" << m_dir << "]\n");
}
CirCache::~CirCache()
@ -681,7 +680,7 @@ public:
const EntryHeaderData& d) {
headoffs = offs;
padsize = d.padsize;
LOGDEB2("CCScanHookRecord::takeone: offs " << (lltodecstr(headoffs)) << " padsize " << (lltodecstr(padsize)) << "\n" );
LOGDEB2("CCScanHookRecord::takeone: offs " << headoffs << " padsize " << padsize << "\n");
return Continue;
}
};
@ -693,7 +692,7 @@ string CirCache::getpath()
bool CirCache::create(int64_t maxsize, int flags)
{
LOGDEB("CirCache::create: [" << (m_dir) << "] maxsz " << (lltodecstr((long long)maxsize)) << " flags 0x" << (flags) << "\n" );
LOGDEB("CirCache::create: [" << m_dir << "] maxsz " << maxsize << " flags 0x" << std::hex << flags <<std::dec<<"\n");
if (m_d == 0) {
LOGERR("CirCache::create: null data\n");
return false;
@ -734,7 +733,9 @@ bool CirCache::create(int64_t maxsize, int flags)
}
m_d->m_maxsize = maxsize;
m_d->m_uniquentries = ((flags & CC_CRUNIQUE) != 0);
LOGDEB2("CirCache::create: rewriting header with maxsize " << (lltodecstr(m_d->m_maxsize)) << " oheadoffs " << (lltodecstr(m_d->m_oheadoffs)) << " nheadoffs " << (lltodecstr(m_d->m_nheadoffs)) << " npadsize " << (m_d->m_npadsize) << " unient " << (int(m_d->m_uniquentries)) << "\n" );
LOGDEB2("CirCache::create: rewriting header with maxsize " << m_d->m_maxsize << " oheadoffs " <<
m_d->m_oheadoffs << " nheadoffs " << m_d->m_nheadoffs << " npadsize " << m_d->m_npadsize <<
" unient " << m_d->m_uniquentries << "\n");
return m_d->writefirstblock();
}
// Else fallthrough to create file
@ -836,7 +837,8 @@ public:
virtual status takeone(int64_t offs, const string& udi,
const EntryHeaderData& d) {
LOGDEB2("Circache:Scan: off " << (long(offs)) << " udi [" << (udi) << "] dcsz " << ((UINT)d.dicsize) << " dtsz " << ((UINT)d.datasize) << " pdsz " << ((UINT)d.padsize) << " flgs " << (d.flags) << "\n" );
LOGDEB2("Circache:Scan: off " << offs << " udi [" << udi << "] dcsz " << d.dicsize << " dtsz " << d.datasize <<
" pdsz " << d.padsize << " flgs " << d.flags << "\n");
if (!m_udi.compare(udi)) {
m_instance++;
m_offs = offs;
@ -858,7 +860,7 @@ bool CirCache::get(const string& udi, string& dic, string *data, int instance)
return false;
}
LOGDEB0("CirCache::get: udi [" << (udi) << "], instance " << (instance) << "\n" );
LOGDEB0("CirCache::get: udi [" << udi << "], instance " << instance << "\n");
// If memory map is up to date, use it:
if (m_d->m_ofskhcplt) {
@ -866,13 +868,13 @@ bool CirCache::get(const string& udi, string& dic, string *data, int instance)
//m_d->khDump();
vector<int64_t> ofss;
if (m_d->khFind(udi, ofss)) {
LOGDEB1("Circache::get: h found, colls " << (ofss.size()) << "\n" );
LOGDEB1("Circache::get: h found, colls " << ofss.size() << "\n");
int finst = 1;
EntryHeaderData d_good;
int64_t o_good = 0;
for (vector<int64_t>::iterator it = ofss.begin();
it != ofss.end(); it++) {
LOGDEB1("Circache::get: trying offs " << ((ULONG)*it) << "\n" );
LOGDEB1("Circache::get: trying offs " << *it << "\n");
EntryHeaderData d;
string fudi;
if (!m_d->readHUdi(*it, d, fudi)) {
@ -894,7 +896,7 @@ bool CirCache::get(const string& udi, string& dic, string *data, int instance)
// Did we read an appropriate entry ?
if (o_good != 0 && (instance == -1 || instance == finst)) {
bool ret = m_d->readDicData(o_good, d_good, dic, data);
LOGDEB0("Circache::get: hfound, " << (chron.millis()) << " mS\n" );
LOGDEB0("Circache::get: hfound, " << chron.millis() << " mS\n");
return ret;
}
// Else try to scan anyway.
@ -913,8 +915,7 @@ bool CirCache::get(const string& udi, string& dic, string *data, int instance)
return false;
}
bool bret = m_d->readDicData(getter.m_offs, getter.m_hd, dic, data);
LOGDEB0("Circache::get: scanfound, " << (chron.millis()) << " mS\n" );
LOGDEB0("Circache::get: scanfound, " << chron.millis() << " mS\n");
return bret;
}
@ -929,7 +930,7 @@ bool CirCache::erase(const string& udi, bool reallyclear)
return false;
}
LOGDEB0("CirCache::erase: udi [" << (udi) << "]\n" );
LOGDEB0("CirCache::erase: udi [" << udi << "]\n");
// If the mem cache is not up to date, update it, we're too lazy
// to do a scan
@ -950,17 +951,17 @@ bool CirCache::erase(const string& udi, bool reallyclear)
}
for (vector<int64_t>::iterator it = ofss.begin(); it != ofss.end(); it++) {
LOGDEB2("CirCache::erase: reading at " << ((unsigned long)*it) << "\n" );
LOGDEB2("CirCache::erase: reading at " << *it << "\n");
EntryHeaderData d;
string fudi;
if (!m_d->readHUdi(*it, d, fudi)) {
return false;
}
LOGDEB2("CirCache::erase: found fudi [" << (fudi) << "]\n" );
LOGDEB2("CirCache::erase: found fudi [" << fudi << "]\n");
if (!fudi.compare(udi)) {
EntryHeaderData nd;
nd.padsize = d.dicsize + d.datasize + d.padsize;
LOGDEB2("CirCache::erase: rewrite at " << ((unsigned long)*it) << "\n" );
LOGDEB2("CirCache::erase: rewrite at " << *it << "\n");
if (*it == m_d->m_nheadoffs) {
m_d->m_npadsize = nd.padsize;
}
@ -988,7 +989,8 @@ public:
virtual status takeone(int64_t offs, const string& udi,
const EntryHeaderData& d) {
LOGDEB2("Circache:ScanSpacer:off " << ((UINT)offs) << " dcsz " << (d.dicsize) << " dtsz " << (d.datasize) << " pdsz " << (d.padsize) << " udi[" << (udi) << "]\n" );
LOGDEB2("Circache:ScanSpacer:off " << offs << " dcsz " << d.dicsize << " dtsz " << d.datasize <<
" pdsz " << d.padsize << " udi[" << udi << "]\n");
sizeseen += CIRCACHE_HEADER_SIZE + d.dicsize + d.datasize + d.padsize;
squashed_udis.push_back(make_pair(udi, offs));
if (sizeseen >= sizewanted) {
@ -1014,7 +1016,7 @@ bool CirCache::put(const string& udi, const ConfSimple *iconf,
string dic;
if (!iconf || !iconf->get("udi", dic) || dic.empty() || dic.compare(udi)) {
m_d->m_reason << "No/bad 'udi' entry in input dic";
LOGERR("Circache::put: no/bad udi: DIC:[" << (dic) << "] UDI [" << (udi) << "]\n" );
LOGERR("Circache::put: no/bad udi: DIC:[" << dic << "] UDI [" << udi << "]\n");
return false;
}
@ -1058,7 +1060,7 @@ bool CirCache::put(const string& udi, const ConfSimple *iconf,
int64_t npadsize = 0;
bool extending = false;
LOGDEB("CirCache::put: nsz " << (nsize) << " oheadoffs " << (m_d->m_oheadoffs) << "\n" );
LOGDEB("CirCache::put: nsz " << nsize << " oheadoffs " << m_d->m_oheadoffs << "\n");
// Check if we can recover some pad space from the (physically) previous
// entry.
@ -1080,7 +1082,7 @@ bool CirCache::put(const string& udi, const ConfSimple *iconf,
// the header, we're going to write on it.
recovpadsize += CIRCACHE_HEADER_SIZE;
} else {
LOGDEB("CirCache::put: recov. prev. padsize " << (pd.padsize) << "\n" );
LOGDEB("CirCache::put: recov. prev. padsize " << pd.padsize << "\n");
pd.padsize = 0;
if (!m_d->writeEntryHeader(m_d->m_nheadoffs, pd)) {
return false;
@ -1093,7 +1095,7 @@ bool CirCache::put(const string& udi, const ConfSimple *iconf,
if (nsize <= recovpadsize) {
// If the new entry fits entirely in the pad area from the
// latest one, no need to recycle stuff
LOGDEB("CirCache::put: new fits in old padsize " << (recovpadsize) << "\n" );
LOGDEB("CirCache::put: new fits in old padsize " << recovpadsize << "\n");
npadsize = recovpadsize - nsize;
} else if (st.st_size < m_d->m_maxsize) {
// Still growing the file.
@ -1103,11 +1105,11 @@ bool CirCache::put(const string& udi, const ConfSimple *iconf,
// Scan the file until we have enough space for the new entry,
// and determine the pad size up to the 1st preserved entry
int64_t scansize = nsize - recovpadsize;
LOGDEB("CirCache::put: scanning for size " << (scansize) << " from offs " << ((UINT)m_d->m_oheadoffs) << "\n" );
LOGDEB("CirCache::put: scanning for size " << scansize << " from offs " << m_d->m_oheadoffs << "\n");
CCScanHookSpacer spacer(scansize);
switch (m_d->scan(m_d->m_oheadoffs, &spacer)) {
case CCScanHook::Stop:
LOGDEB("CirCache::put: Scan ok, sizeseen " << (spacer.sizeseen) << "\n" );
LOGDEB("CirCache::put: Scan ok, sizeseen " << spacer.sizeseen << "\n");
npadsize = spacer.sizeseen - scansize;
break;
case CCScanHook::Eof:
@ -1122,7 +1124,7 @@ bool CirCache::put(const string& udi, const ConfSimple *iconf,
m_d->khClear(spacer.squashed_udis);
}
LOGDEB("CirCache::put: writing " << (nsize) << " at " << (nwriteoffs) << " padsize " << (npadsize) << "\n" );
LOGDEB("CirCache::put: writing " << nsize << " at " << nwriteoffs << " padsize " << npadsize << "\n");
if (lseek(m_d->m_fd, nwriteoffs, 0) != nwriteoffs) {
m_d->m_reason << "CirCache::put: lseek failed: " << errno;