tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

got rid of off_t

Browse Source
This commit is contained in:
Jean-Francois Dockes
2017-02-28 20:36:01 +01:00
parent 9a9608e00b
commit bde991c08a
17 changed files with 95 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/circache.cpp
+47 -47
View File
@@ -145,7 +145,7 @@ class CCScanHook {
public:
virtual ~CCScanHook() {}
enum status {Stop, Continue, Error, Eof};
virtual status takeone(off_t offs, const string& udi,
virtual status takeone(int64_t offs, const string& udi,
const EntryHeaderData& d) = 0;
};
@@ -197,22 +197,22 @@ public:
return false;
}
};
typedef multimap<UdiH, off_t> kh_type;
typedef multimap<UdiH, off_t>::value_type kh_value_type;
typedef multimap<UdiH, int64_t> kh_type;
typedef multimap<UdiH, int64_t>::value_type kh_value_type;
class CirCacheInternal {
public:
int m_fd;
////// These are cache persistent state and written to the first block:
// Maximum file size, after which we begin reusing old space
off_t m_maxsize;
int64_t m_maxsize;
// Offset of the oldest header, or max file offset (file size)
// while the file is growing. This is the next write position.
off_t m_oheadoffs;
int64_t m_oheadoffs;
// Offset of last write (newest header)
off_t m_nheadoffs;
int64_t m_nheadoffs;
// Pad size for newest entry.
off_t m_npadsize;
int64_t m_npadsize;
// Keep history or only last entry
bool m_uniquentries;
///////////////////// End header entries
@@ -226,7 +226,7 @@ public:
// State for rewind/next/getcurrent operation. This could/should
// be moved to a separate iterator.
off_t m_itoffs;
int64_t m_itoffs;
EntryHeaderData m_ithd;
// Offset cache
@@ -234,7 +234,7 @@ public:
bool m_ofskhcplt; // Has cache been fully read since open?
// Add udi->offset translation to map
bool khEnter(const string& udi, off_t ofs) {
bool khEnter(const string& udi, int64_t ofs) {
UdiH h(udi);
LOGDEB2("Circache::khEnter: h " << (h.asHexString()) << " offs " << ((ULONG)ofs) << " udi [" << (udi) << "]\n" );
@@ -265,7 +265,7 @@ public:
// Return vector of candidate offsets for udi (possibly several
// because there may be hash collisions, and also multiple
// instances).
bool khFind(const string& udi, vector<off_t>& ofss) {
bool khFind(const string& udi, vector<int64_t>& ofss) {
ofss.clear();
UdiH h(udi);
@@ -295,7 +295,7 @@ public:
return true;
}
// Clear entry for udi/offs
bool khClear(const pair<string, off_t>& ref) {
bool khClear(const pair<string, int64_t>& ref) {
UdiH h(ref.first);
pair<kh_type::iterator, kh_type::iterator> p = m_ofskh.equal_range(h);
if (p.first != m_ofskh.end() && (p.first->first == h)) {
@@ -309,8 +309,8 @@ public:
return true;
}
// Clear entries for vector of udi/offs
bool khClear(const vector<pair<string, off_t> >& udis) {
for (vector<pair<string, off_t> >::const_iterator it = udis.begin();
bool khClear(const vector<pair<string, int64_t> >& udis) {
for (vector<pair<string, int64_t> >::const_iterator it = udis.begin();
it != udis.end(); it++) {
khClear(*it);
}
@@ -434,7 +434,7 @@ public:
return true;
}
bool writeEntryHeader(off_t offset, const EntryHeaderData& d,
bool writeEntryHeader(int64_t offset, const EntryHeaderData& d,
bool eraseData = false) {
if (m_fd < 0) {
m_reason << "writeEntryHeader: not open ";
@@ -467,7 +467,7 @@ public:
return true;
}
CCScanHook::status readEntryHeader(off_t offset, EntryHeaderData& d) {
CCScanHook::status readEntryHeader(int64_t offset, EntryHeaderData& d) {
if (m_fd < 0) {
m_reason << "readEntryHeader: not open ";
return CCScanHook::Error;
@@ -500,14 +500,14 @@ public:
return CCScanHook::Continue;
}
CCScanHook::status scan(off_t startoffset, CCScanHook *user,
CCScanHook::status scan(int64_t startoffset, CCScanHook *user,
bool fold = false) {
if (m_fd < 0) {
m_reason << "scan: not open ";
return CCScanHook::Error;
}
off_t so0 = startoffset;
int64_t so0 = startoffset;
bool already_folded = false;
while (true) {
@@ -569,7 +569,7 @@ public:
}
}
bool readHUdi(off_t hoffs, EntryHeaderData& d, string& udi) {
bool readHUdi(int64_t hoffs, EntryHeaderData& d, string& udi) {
if (readEntryHeader(hoffs, d) != CCScanHook::Continue) {
return false;
}
@@ -590,9 +590,9 @@ public:
return true;
}
bool readDicData(off_t hoffs, EntryHeaderData& hd, string& dic,
bool readDicData(int64_t hoffs, EntryHeaderData& hd, string& dic,
string* data) {
off_t offs = hoffs + CIRCACHE_HEADER_SIZE;
int64_t offs = hoffs + CIRCACHE_HEADER_SIZE;
// This syscall could be avoided in some cases if we saved the offset
// at each seek. In most cases, we just read the header and we are
// at the right position
@@ -674,12 +674,12 @@ string CirCache::getReason()
// physical record in the file
class CCScanHookRecord : public CCScanHook {
public:
off_t headoffs;
off_t padsize;
int64_t headoffs;
int64_t padsize;
CCScanHookRecord()
: headoffs(0), padsize(0) {
}
virtual status takeone(off_t offs, const string& udi,
virtual status takeone(int64_t offs, const string& udi,
const EntryHeaderData& d) {
headoffs = offs;
padsize = d.padsize;
@@ -693,7 +693,7 @@ string CirCache::getpath()
return m_d->datafn(m_dir);
}
bool CirCache::create(off_t maxsize, int flags)
bool CirCache::create(int64_t maxsize, int flags)
{
LOGDEB("CirCache::create: [" << (m_dir) << "] maxsz " << (lltodecstr((long long)maxsize)) << " flags 0x" << (flags) << "\n" );
if (m_d == 0) {
@@ -787,7 +787,7 @@ bool CirCache::open(OpMode mode)
class CCScanHookDump : public CCScanHook {
public:
virtual status takeone(off_t offs, const string& udi,
virtual status takeone(int64_t offs, const string& udi,
const EntryHeaderData& d) {
cout << "Scan: offs " << offs << " dicsize " << d.dicsize
<< " datasize " << d.datasize << " padsize " << d.padsize <<
@@ -803,7 +803,7 @@ bool CirCache::dump()
// Start at oldest header. This is eof while the file is growing, scan will
// fold to bot at once.
off_t start = m_d->m_oheadoffs;
int64_t start = m_d->m_oheadoffs;
switch (m_d->scan(start, &dumper, true)) {
case CCScanHook::Stop:
@@ -830,13 +830,13 @@ public:
string m_udi;
int m_targinstance;
int m_instance;
off_t m_offs;
int64_t m_offs;
EntryHeaderData m_hd;
CCScanHookGetter(const string& udi, int ti)
: m_udi(udi), m_targinstance(ti), m_instance(0), m_offs(0) {}
virtual status takeone(off_t offs, const string& udi,
virtual status takeone(int64_t offs, const string& udi,
const EntryHeaderData& d) {
LOGDEB2("Circache:Scan: off " << (long(offs)) << " udi [" << (udi) << "] dcsz " << ((UINT)d.dicsize) << " dtsz " << ((UINT)d.datasize) << " pdsz " << ((UINT)d.padsize) << " flgs " << (d.flags) << "\n" );
if (!m_udi.compare(udi)) {
@@ -866,13 +866,13 @@ bool CirCache::get(const string& udi, string& dic, string *data, int instance)
if (m_d->m_ofskhcplt) {
LOGDEB1("CirCache::get: using ofskh\n" );
//m_d->khDump();
vector<off_t> ofss;
vector<int64_t> ofss;
if (m_d->khFind(udi, ofss)) {
LOGDEB1("Circache::get: h found, colls " << (ofss.size()) << "\n" );
int finst = 1;
EntryHeaderData d_good;
off_t o_good = 0;
for (vector<off_t>::iterator it = ofss.begin();
int64_t o_good = 0;
for (vector<int64_t>::iterator it = ofss.begin();
it != ofss.end(); it++) {
LOGDEB1("Circache::get: trying offs " << ((ULONG)*it) << "\n" );
EntryHeaderData d;
@@ -904,7 +904,7 @@ bool CirCache::get(const string& udi, string& dic, string *data, int instance)
}
CCScanHookGetter getter(udi, instance);
off_t start = m_d->m_oheadoffs;
int64_t start = m_d->m_oheadoffs;
CCScanHook::status ret = m_d->scan(start, &getter, true);
if (ret == CCScanHook::Eof) {
@@ -944,14 +944,14 @@ bool CirCache::erase(const string& udi, bool reallyclear)
}
}
vector<off_t> ofss;
vector<int64_t> ofss;
if (!m_d->khFind(udi, ofss)) {
// Udi not in there, erase ok
LOGDEB("CirCache::erase: khFind returns none\n" );
return true;
}
for (vector<off_t>::iterator it = ofss.begin(); it != ofss.end(); it++) {
for (vector<int64_t>::iterator it = ofss.begin(); it != ofss.end(); it++) {
LOGDEB2("CirCache::erase: reading at " << ((unsigned long)*it) << "\n" );
EntryHeaderData d;
string fudi;
@@ -980,15 +980,15 @@ bool CirCache::erase(const string& udi, bool reallyclear)
// entry.
class CCScanHookSpacer : public CCScanHook {
public:
off_t sizewanted;
off_t sizeseen;
vector<pair<string, off_t> > squashed_udis;
CCScanHookSpacer(off_t sz)
int64_t sizewanted;
int64_t sizeseen;
vector<pair<string, int64_t> > squashed_udis;
CCScanHookSpacer(int64_t sz)
: sizewanted(sz), sizeseen(0) {
assert(sz > 0);
}
virtual status takeone(off_t offs, const string& udi,
virtual status takeone(int64_t offs, const string& udi,
const EntryHeaderData& d) {
LOGDEB2("Circache:ScanSpacer:off " << ((UINT)offs) << " dcsz " << (d.dicsize) << " dtsz " << (d.datasize) << " pdsz " << (d.padsize) << " udi[" << (udi) << "]\n" );
sizeseen += CIRCACHE_HEADER_SIZE + d.dicsize + d.datasize + d.padsize;
@@ -1058,16 +1058,16 @@ bool CirCache::put(const string& udi, const ConfSimple *iconf,
}
// Characteristics for the new entry.
off_t nsize = CIRCACHE_HEADER_SIZE + dic.size() + datalen;
off_t nwriteoffs = m_d->m_oheadoffs;
off_t npadsize = 0;
int64_t nsize = CIRCACHE_HEADER_SIZE + dic.size() + datalen;
int64_t nwriteoffs = m_d->m_oheadoffs;
int64_t npadsize = 0;
bool extending = false;
LOGDEB("CirCache::put: nsz " << (nsize) << " oheadoffs " << (m_d->m_oheadoffs) << "\n" );
// Check if we can recover some pad space from the (physically) previous
// entry.
off_t recovpadsize = m_d->m_oheadoffs == CIRCACHE_FIRSTBLOCK_SIZE ?
int64_t recovpadsize = m_d->m_oheadoffs == CIRCACHE_FIRSTBLOCK_SIZE ?
0 : m_d->m_npadsize;
if (recovpadsize != 0) {
// Need to read the latest entry's header, to rewrite it with a
@@ -1107,7 +1107,7 @@ bool CirCache::put(const string& udi, const ConfSimple *iconf,
} else {
// Scan the file until we have enough space for the new entry,
// and determine the pad size up to the 1st preserved entry
off_t scansize = nsize - recovpadsize;
int64_t scansize = nsize - recovpadsize;
LOGDEB("CirCache::put: scanning for size " << (scansize) << " from offs " << ((UINT)m_d->m_oheadoffs) << "\n" );
CCScanHookSpacer spacer(scansize);
switch (m_d->scan(m_d->m_oheadoffs, &spacer)) {
@@ -1177,8 +1177,8 @@ bool CirCache::rewind(bool& eof)
eof = false;
off_t fsize = lseek(m_d->m_fd, 0, SEEK_END);
if (fsize == (off_t) - 1) {
int64_t fsize = lseek(m_d->m_fd, 0, SEEK_END);
if (fsize == (int64_t) - 1) {
LOGERR("CirCache::rewind: seek to EOF failed\n" );
return false;
}
@@ -1564,7 +1564,7 @@ b1:
if (argc != 1) {
Usage();
}
off_t sizekb = atoi(*argv++);
int64_t sizekb = atoi(*argv++);
argc--;
int flags = 0;
if (op_flags & OPT_u) {
src/utils/circache.h
+2 -1
View File
@@ -37,6 +37,7 @@
*/
#include <sys/types.h>
#include <stdint.h>
#include <string>
@@ -57,7 +58,7 @@ public:
// Truncate file (restart from scratch).
CC_CRTRUNCATE = 2
};
virtual bool create(off_t maxsize, int flags);
virtual bool create(int64_t maxsize, int flags);
enum OpMode {CC_OPREAD, CC_OPWRITE};
virtual bool open(OpMode mode);
src/utils/execmd.cpp
+10 -7
View File
@@ -298,15 +298,18 @@ public:
m_parent->reset();
}
private:
ExecCmd::Internal *m_parent;
bool m_active;
ExecCmd::Internal *m_parent{nullptr};
bool m_active{false};
};
ExecCmd::~ExecCmd()
{
ExecCmdRsrc(this->m);
if (m) {
ExecCmdRsrc r(m);
}
if (m) {
delete m;
m = nullptr;
}
}
@@ -455,7 +458,7 @@ int ExecCmd::startExec(const string& cmd, const vector<string>& args,
}
// The resource manager ensures resources are freed if we return early
ExecCmdRsrc e(this->m);
ExecCmdRsrc e(m);
if (has_input && pipe(m->m_pipein) < 0) {
LOGERR("ExecCmd::startExec: pipe(2) failed. errno " << errno << "\n" );
@@ -746,7 +749,7 @@ int ExecCmd::doexec(const string& cmd, const vector<string>& args,
}
// Cleanup in case we return early
ExecCmdRsrc e(this->m);
ExecCmdRsrc e(m);
SelectLoop myloop;
int ret = 0;
if (input || output) {
@@ -948,7 +951,7 @@ int ExecCmd::getline(string& data, int timeosecs)
// overhead.
int ExecCmd::wait()
{
ExecCmdRsrc e(this->m);
ExecCmdRsrc e(m);
int status = -1;
if (!m->m_killRequest && m->m_pid > 0) {
if (waitpid(m->m_pid, &status, 0) < 0) {
@@ -964,7 +967,7 @@ int ExecCmd::wait()
bool ExecCmd::maybereap(int *status)
{
ExecCmdRsrc e(this->m);
ExecCmdRsrc e(m);
*status = -1;
if (m->m_pid <= 0) {
src/utils/readfile.cpp
+4 -4
View File
@@ -65,7 +65,7 @@ bool file_to_string(const string& fn, string& data, string *reason)
{
return file_to_string(fn, data, 0, size_t(-1), reason);
}
bool file_to_string(const string& fn, string& data, off_t offs, size_t cnt,
bool file_to_string(const string& fn, string& data, int64_t offs, size_t cnt,
string *reason)
{
FileToString accum(data);
@@ -82,7 +82,7 @@ const int RDBUFSZ = 8192;
// on both linux i586 and macosx (compared to just append())
// Also tried a version with mmap, but it's actually slower on the mac and not
// faster on linux.
bool file_scan(const string& fn, FileScanDo* doer, off_t startoffs,
bool file_scan(const string& fn, FileScanDo* doer, int64_t startoffs,
size_t cnttoread, string *reason)
{
if (startoffs < 0) {
@@ -121,7 +121,7 @@ bool file_scan(const string& fn, FileScanDo* doer, off_t startoffs,
doer->init(0, reason);
}
off_t curoffs = 0;
int64_t curoffs = 0;
if (startoffs > 0 && !fn.empty()) {
if (lseek(fd, startoffs, SEEK_SET) != startoffs) {
catstrerror(reason, "lseek", errno);
@@ -233,7 +233,7 @@ Usage(void)
int main(int argc, const char **argv)
{
off_t offs = 0;
int64_t offs = 0;
size_t cnt = size_t(-1);
thisprog = argv[0];
argc--;
src/utils/readfile.h
+2 -2
View File
@@ -34,7 +34,7 @@ public:
bool file_scan(const std::string& filename, FileScanDo* doer, std::string *reason = 0);
/* Same but only process count cnt from offset offs. Set cnt to size_t(-1)
* for no limit */
bool file_scan(const std::string& fn, FileScanDo* doer, off_t offs, size_t cnt,
bool file_scan(const std::string& fn, FileScanDo* doer, int64_t offs, size_t cnt,
std::string *reason = 0);
/**
@@ -45,6 +45,6 @@ bool file_to_string(const std::string& filename, std::string& data, std::string
/** Read file chunk into string. Set cnt to size_t(-1) for whole file */
bool file_to_string(const std::string& filename, std::string& data,
off_t offs, size_t cnt, std::string *reason = 0);
int64_t offs, size_t cnt, std::string *reason = 0);
#endif /* _READFILE_H_INCLUDED_ */
src/utils/smallut.cpp
+1 -1
View File
@@ -763,7 +763,7 @@ string displayableBytes(int64_t size)
unit = " GB ";
roundable = double(size) / 1E9;
}
size = off_t(round(roundable));
size = int64_t(round(roundable));
return lltodecstr(size).append(unit);
}