tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

remove struct stat from interfaces

Browse Source
This commit is contained in:
Jean-Francois Dockes 2020-03-31 11:17:07 +02:00
parent b5086139aa
commit 34d23589a2
23 changed files with 233 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/index/fetcher.h
View File

@ -17,11 +17,11 @@
#ifndef _FETCHER_H_INCLUDED_ #ifndef _FETCHER_H_INCLUDED_
#define _FETCHER_H_INCLUDED_ #define _FETCHER_H_INCLUDED_
#include "safesysstat.h"
#include <string> #include <string>
#include <memory> #include <memory>
#include "rcldoc.h" #include "rcldoc.h"
#include "pathut.h"
class RclConfig; class RclConfig;
@ -51,7 +51,7 @@ public:
enum RawDocKind {RDK_FILENAME, RDK_DATA, RDK_DATADIRECT}; enum RawDocKind {RDK_FILENAME, RDK_DATA, RDK_DATADIRECT};
RawDocKind kind; RawDocKind kind;
std::string data; // Doc data or file name std::string data; // Doc data or file name
struct stat st; // Only used if RDK_FILENAME struct PathStat st; // Only used if RDK_FILENAME
}; };
/** /**

7
src/index/fsfetcher.cpp
View File

@ -17,7 +17,6 @@
#include "autoconfig.h" #include "autoconfig.h"
#include <errno.h> #include <errno.h>
#include "safesysstat.h"
#include "log.h" #include "log.h"
#include "cstr.h" #include "cstr.h"
@ -29,7 +28,7 @@
using std::string; using std::string;
static DocFetcher::Reason urltopath(RclConfig* cnf, const Rcl::Doc& idoc, static DocFetcher::Reason urltopath(RclConfig* cnf, const Rcl::Doc& idoc,
string& fn, struct stat& st) string& fn, struct PathStat& st)
{ {
// The url has to be like file:// // The url has to be like file://
fn = fileurltolocalpath(idoc.url); fn = fileurltolocalpath(idoc.url);
@ -62,7 +61,7 @@ bool FSDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
bool FSDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig) bool FSDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig)
{ {
string fn; string fn;
struct stat st; struct PathStat st;
if (urltopath(cnf, idoc, fn, st) != DocFetcher::FetchOk) if (urltopath(cnf, idoc, fn, st) != DocFetcher::FetchOk)
return false; return false;
FsIndexer::makesig(&st, sig); FsIndexer::makesig(&st, sig);
@ -72,7 +71,7 @@ bool FSDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig)
DocFetcher::Reason FSDocFetcher::testAccess(RclConfig* cnf, const Rcl::Doc& idoc) DocFetcher::Reason FSDocFetcher::testAccess(RclConfig* cnf, const Rcl::Doc& idoc)
{ {
string fn; string fn;
struct stat st; struct PathStat st;
DocFetcher::Reason reason = urltopath(cnf, idoc, fn, st); DocFetcher::Reason reason = urltopath(cnf, idoc, fn, st);
if (reason != DocFetcher::FetchOk) { if (reason != DocFetcher::FetchOk) {
return reason; return reason;

30
src/index/fsindexer.cpp
View File

@ -21,7 +21,6 @@
#include <stdio.h> #include <stdio.h>
#include <errno.h> #include <errno.h>
#include <cstring> #include <cstring>
#include "safesysstat.h"
#include <iostream> #include <iostream>
#include <list> #include <list>
@ -69,14 +68,14 @@ extern void *FsIndexerDbUpdWorker(void*);
class InternfileTask { class InternfileTask {
public: public:
// Take some care to avoid sharing string data (if string impl is cow) // Take some care to avoid sharing string data (if string impl is cow)
InternfileTask(const std::string &f, const struct stat *i_stp, InternfileTask(const std::string &f, const struct PathStat *i_stp,
map<string,string> lfields) map<string,string> lfields)
: fn(f.begin(), f.end()), statbuf(*i_stp) : fn(f.begin(), f.end()), statbuf(*i_stp)
{ {
map_ss_cp_noshr(lfields, &localfields); map_ss_cp_noshr(lfields, &localfields);
} }
string fn; string fn;
struct stat statbuf; struct PathStat statbuf;
map<string,string> localfields; map<string,string> localfields;
}; };
extern void *FsIndexerInternfileWorker(void*); extern void *FsIndexerInternfileWorker(void*);
@ -369,7 +368,7 @@ bool FsIndexer::indexFiles(list<string>& files, int flags)
continue; continue;
} }
struct stat stb; struct PathStat stb;
int ststat = path_fileprops(*it, &stb, follow); int ststat = path_fileprops(*it, &stb, follow);
if (ststat != 0) { if (ststat != 0) {
LOGERR("FsIndexer::indexFiles: (l)stat " << *it << ": " << LOGERR("FsIndexer::indexFiles: (l)stat " << *it << ": " <<
@ -378,7 +377,8 @@ bool FsIndexer::indexFiles(list<string>& files, int flags)
continue; continue;
} }
if (!(flags & ConfIndexer::IxFIgnoreSkip) && if (!(flags & ConfIndexer::IxFIgnoreSkip) &&
(S_ISREG(stb.st_mode) || S_ISLNK(stb.st_mode))) { (stb.pst_type == PathStat::PST_REGULAR ||
stb.pst_type == PathStat::PST_SYMLINK)) {
if (!walker.inOnlyNames(path_getsimple(*it))) { if (!walker.inOnlyNames(path_getsimple(*it))) {
it++; it++;
continue; continue;
@ -499,10 +499,10 @@ void FsIndexer::setlocalfields(const map<string, string>& fields, Rcl::Doc& doc)
} }
} }
void FsIndexer::makesig(const struct stat *stp, string& out) void FsIndexer::makesig(const struct PathStat *stp, string& out)
{ {
out = lltodecstr(stp->st_size) + out = lltodecstr(stp->pst_size) +
lltodecstr(o_uptodate_test_use_mtime ? stp->st_mtime : stp->st_ctime); lltodecstr(o_uptodate_test_use_mtime ? stp->pst_mtime : stp->pst_ctime);
} }
#ifdef IDX_THREADS #ifdef IDX_THREADS
@ -572,7 +572,7 @@ void *FsIndexerInternfileWorker(void * fsp)
/// the actual indexing work. The Rcl::Doc created by internfile() /// the actual indexing work. The Rcl::Doc created by internfile()
/// mostly contains pretty raw utf8 data. /// mostly contains pretty raw utf8 data.
FsTreeWalker::Status FsTreeWalker::Status
FsIndexer::processone(const std::string &fn, const struct stat *stp, FsIndexer::processone(const std::string &fn, const struct PathStat *stp,
FsTreeWalker::CbFlag flg) FsTreeWalker::CbFlag flg)
{ {
if (m_updater) { if (m_updater) {
@ -633,7 +633,7 @@ bool FsIndexer::launchAddOrUpdate(const string& udi, const string& parent_udi,
FsTreeWalker::Status FsTreeWalker::Status
FsIndexer::processonefile(RclConfig *config, FsIndexer::processonefile(RclConfig *config,
const std::string &fn, const struct stat *stp, const std::string &fn, const struct PathStat *stp,
const map<string, string>& localfields) const map<string, string>& localfields)
{ {
//////////////////// ////////////////////
@ -671,7 +671,7 @@ FsIndexer::processonefile(RclConfig *config,
// miss the data update. We would have to store both the mtime and // miss the data update. We would have to store both the mtime and
// the ctime to avoid this // the ctime to avoid this
bool xattronly = m_detectxattronly && !m_db->inFullReset() && bool xattronly = m_detectxattronly && !m_db->inFullReset() &&
existingDoc && needupdate && (stp->st_mtime < stp->st_ctime); existingDoc && needupdate && (stp->pst_mtime < stp->pst_ctime);
LOGDEB("processone: needupdate " << needupdate << " noretry " << LOGDEB("processone: needupdate " << needupdate << " noretry " <<
m_noretryfailed << " existing " << existingDoc << " oldsig [" << m_noretryfailed << " existing " << existingDoc << " oldsig [" <<
@ -708,7 +708,7 @@ FsIndexer::processonefile(RclConfig *config,
} }
LOGDEB0("processone: processing: [" << LOGDEB0("processone: processing: [" <<
displayableBytes(stp->st_size) << "] " << fn << "\n"); displayableBytes(stp->pst_size) << "] " << fn << "\n");
// Note that we used to do the full path here, but I ended up // Note that we used to do the full path here, but I ended up
// believing that it made more sense to use only the file name // believing that it made more sense to use only the file name
@ -720,7 +720,7 @@ FsIndexer::processonefile(RclConfig *config,
Rcl::Doc doc; Rcl::Doc doc;
char ascdate[30]; char ascdate[30];
sprintf(ascdate, "%ld", long(stp->st_mtime)); sprintf(ascdate, "%ld", long(stp->pst_mtime));
bool hadNullIpath = false; bool hadNullIpath = false;
string mimetype; string mimetype;
@ -789,7 +789,7 @@ FsIndexer::processonefile(RclConfig *config,
// Set container file name for all docs, top or subdoc // Set container file name for all docs, top or subdoc
doc.meta[Rcl::Doc::keytcfn] = utf8fn; doc.meta[Rcl::Doc::keytcfn] = utf8fn;
doc.pcbytes = lltodecstr(stp->st_size); doc.pcbytes = lltodecstr(stp->pst_size);
// Document signature for up to date checks. All subdocs inherit the // Document signature for up to date checks. All subdocs inherit the
// file's. // file's.
doc.sig = sig; doc.sig = sig;
@ -880,7 +880,7 @@ FsIndexer::processonefile(RclConfig *config,
fileDoc.url = path_pathtofileurl(fn); fileDoc.url = path_pathtofileurl(fn);
if (m_havelocalfields) if (m_havelocalfields)
setlocalfields(localfields, fileDoc); setlocalfields(localfields, fileDoc);
fileDoc.pcbytes = lltodecstr(stp->st_size); fileDoc.pcbytes = lltodecstr(stp->pst_size);
} }
fileDoc.sig = sig; fileDoc.sig = sig;

9
src/index/fsindexer.h
View File

@ -28,7 +28,7 @@
class DbIxStatusUpdater; class DbIxStatusUpdater;
class FIMissingStore; class FIMissingStore;
struct stat; struct PathStat;
class DbUpdTask; class DbUpdTask;
class InternfileTask; class InternfileTask;
@ -75,10 +75,10 @@ public:
/** Tree walker callback method */ /** Tree walker callback method */
FsTreeWalker::Status FsTreeWalker::Status
processone(const string &fn, const struct stat *, FsTreeWalker::CbFlag); processone(const string &fn, const struct PathStat *, FsTreeWalker::CbFlag);
/** Make signature for file up to date checks */ /** Make signature for file up to date checks */
static void makesig(const struct stat *stp, string& out); static void makesig(const struct PathStat *stp, string& out);
private: private:
@ -160,7 +160,8 @@ private:
string getDbDir() {return m_config->getDbDir();} string getDbDir() {return m_config->getDbDir();}
FsTreeWalker::Status FsTreeWalker::Status
processonefile(RclConfig *config, const string &fn, processonefile(RclConfig *config, const string &fn,
const struct stat *, const map<string,string>& localfields); const struct PathStat *,
const map<string,string>& localfields);
}; };
#endif /* _fsindexer_h_included_ */ #endif /* _fsindexer_h_included_ */

5
src/index/indexer.cpp
View File

@ -33,7 +33,6 @@
#include "pathut.h" #include "pathut.h"
#include "idxstatus.h" #include "idxstatus.h"
#include "execmd.h" #include "execmd.h"
#include "safesysstat.h"
#ifdef RCL_USE_ASPELL #ifdef RCL_USE_ASPELL
#include "rclaspell.h" #include "rclaspell.h"
@ -86,9 +85,9 @@ bool runWebFilesMoverScript(RclConfig *config)
/* Arrange to not actually run the script if the directory did not change */ /* Arrange to not actually run the script if the directory did not change */
static time_t dirmtime; static time_t dirmtime;
time_t ndirmtime = 0; time_t ndirmtime = 0;
struct stat st; struct PathStat st;
if (path_fileprops(downloadsdir.c_str(), &st) == 0) { if (path_fileprops(downloadsdir.c_str(), &st) == 0) {
ndirmtime = st.st_mtime; ndirmtime = st.pst_mtime;
} }
/* If stat fails, presumably Downloads does not exist or is not /* If stat fails, presumably Downloads does not exist or is not
accessible, dirmtime and mdirmtime stay at 0, and we never accessible, dirmtime and mdirmtime stay at 0, and we never

59
src/index/mimetype.cpp
View File

@ -15,11 +15,8 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/ */
#ifndef TEST_MIMETYPE
#include "autoconfig.h" #include "autoconfig.h"
#include "safesysstat.h"
#include <ctype.h> #include <ctype.h>
#include <string> #include <string>
#include <list> #include <list>
@ -139,7 +136,7 @@ static string mimetypefromdata(RclConfig *cfg, const string &fn, bool usfc)
/// Guess mime type, first from suffix, then from file data. We also /// Guess mime type, first from suffix, then from file data. We also
/// have a list of suffixes that we don't touch at all. /// have a list of suffixes that we don't touch at all.
string mimetype(const string &fn, const struct stat *stp, string mimetype(const string &fn, const struct PathStat *stp,
RclConfig *cfg, bool usfc) RclConfig *cfg, bool usfc)
{ {
// Use stat data if available to check for non regular files // Use stat data if available to check for non regular files
@ -149,14 +146,14 @@ string mimetype(const string &fn, const struct stat *stp,
// comes from bsd. Thos may surprise a user trying to use a // comes from bsd. Thos may surprise a user trying to use a
// 'mime:' filter with the query language, but it's not work // 'mime:' filter with the query language, but it's not work
// changing (would force a reindex). // changing (would force a reindex).
if (S_ISDIR(stp->st_mode)) if (stp->pst_type == PathStat::PST_DIR)
return "inode/directory"; return "inode/directory";
if (S_ISLNK(stp->st_mode)) if (stp->pst_type == PathStat::PST_SYMLINK)
return "inode/symlink"; return "inode/symlink";
if (!S_ISREG(stp->st_mode)) if (stp->pst_type != PathStat::PST_REGULAR)
return "inode/x-fsspecial"; return "inode/x-fsspecial";
// Empty files are just this: avoid further errors with actual filters. // Empty files are just this: avoid further errors with actual filters.
if (stp->st_size == 0) if (stp->pst_size == 0)
return "inode/x-empty"; return "inode/x-empty";
} }
@ -203,49 +200,3 @@ string mimetype(const string &fn, const struct stat *stp,
return mtype; return mtype;
} }
#else // TEST->
#include <stdio.h>
#include "safesysstat.h"
#include <cstdlib>
#include <iostream>
#include "log.h"
#include "rclconfig.h"
#include "rclinit.h"
#include "mimetype.h"
using namespace std;
int main(int argc, const char **argv)
{
string reason;
RclConfig *config = recollinit(0, 0, 0, reason);
if (config == 0 || !config->ok()) {
string str = "Configuration problem: ";
str += reason;
fprintf(stderr, "%s\n", str.c_str());
exit(1);
}
while (--argc > 0) {
string filename = *++argv;
struct stat st;
if (lstat(filename.c_str(), &st)) {
fprintf(stderr, "Can't stat %s\n", filename.c_str());
continue;
}
cout << filename << " -> " <<
mimetype(filename, &st, config, true) << endl;
}
return 0;
}
#endif // TEST

4
src/index/mimetype.h
View File

@ -17,10 +17,10 @@
#ifndef _MIMETYPE_H_INCLUDED_ #ifndef _MIMETYPE_H_INCLUDED_
#define _MIMETYPE_H_INCLUDED_ #define _MIMETYPE_H_INCLUDED_
#include "safesysstat.h"
#include <string> #include <string>
class RclConfig; class RclConfig;
struct PathStat;
/** /**
* Try to determine a mime type for file. * Try to determine a mime type for file.
@ -32,7 +32,7 @@ class RclConfig;
* @param cfg recoll config * @param cfg recoll config
* @param usfc Use system's 'file' command as last resort (or not) * @param usfc Use system's 'file' command as last resort (or not)
*/ */
std::string mimetype(const std::string &filename, const struct stat *stp, std::string mimetype(const std::string &filename, const struct PathStat *stp,
RclConfig *cfg, bool usfc); RclConfig *cfg, bool usfc);

10
src/index/rclmonrcv.cpp
View File

@ -21,7 +21,6 @@
#include <errno.h> #include <errno.h>
#include <cstdio> #include <cstdio>
#include <cstring> #include <cstring>
#include "safesysstat.h"
#include "safeunistd.h" #include "safeunistd.h"
#include "log.h" #include "log.h"
@ -74,7 +73,7 @@ public:
virtual ~WalkCB() {} virtual ~WalkCB() {}
virtual FsTreeWalker::Status virtual FsTreeWalker::Status
processone(const string &fn, const struct stat *st, processone(const string &fn, const struct PathStat *st,
FsTreeWalker::CbFlag flg) { FsTreeWalker::CbFlag flg) {
MONDEB("rclMonRcvRun: processone " << fn << " m_mon " << m_mon << MONDEB("rclMonRcvRun: processone " << fn << " m_mon " << m_mon <<
" m_mon->ok " << (m_mon ? m_mon->ok() : false) << std::endl); " m_mon->ok " << (m_mon ? m_mon->ok() : false) << std::endl);
@ -182,12 +181,7 @@ void *rclMonRcvRun(void *q)
} }
// We have to special-case regular files which are part of the topdirs // We have to special-case regular files which are part of the topdirs
// list because we the tree walker only adds watches for directories // list because we the tree walker only adds watches for directories
struct stat st; if (path_isdir(*it, follow)) {
if (path_fileprops(*it, &st, follow) != 0) {
LOGERR("rclMonRcvRun: stat failed for " << *it << "\n");
continue;
}
if (S_ISDIR(st.st_mode)) {
LOGDEB("rclMonRcvRun: walking " << *it << "\n"); LOGDEB("rclMonRcvRun: walking " << *it << "\n");
if (walker.walk(*it, walkcb) != FsTreeWalker::FtwOk) { if (walker.walk(*it, walkcb) != FsTreeWalker::FtwOk) {
LOGERR("rclMonRcvRun: tree walk failed\n"); LOGERR("rclMonRcvRun: tree walk failed\n");

5
src/index/recollindex.cpp
View File

@ -26,7 +26,6 @@
#else #else
#include <direct.h> #include <direct.h>
#endif #endif
#include "safesysstat.h"
#include "safefcntl.h" #include "safefcntl.h"
#include "safeunistd.h" #include "safeunistd.h"
@ -264,8 +263,8 @@ public:
{ {
} }
virtual FsTreeWalker::Status virtual FsTreeWalker::Status
processone(const string& fn, const struct stat *, FsTreeWalker::CbFlag flg) processone(const string& fn, const struct PathStat *,
{ FsTreeWalker::CbFlag flg) {
if (flg== FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwRegular){ if (flg== FsTreeWalker::FtwDirEnter || flg == FsTreeWalker::FtwRegular){
if (m_pats.empty()) { if (m_pats.empty()) {
cerr << "Selecting " << fn << endl; cerr << "Selecting " << fn << endl;

21
src/index/webqueue.cpp
View File

@ -20,7 +20,6 @@
#include <string.h> #include <string.h>
#include <errno.h> #include <errno.h>
#include "safesysstat.h"
#include "safeunistd.h" #include "safeunistd.h"
#include "cstr.h" #include "cstr.h"
@ -322,13 +321,13 @@ bool WebQueueIndexer::indexFiles(list<string>& files)
LOGERR("WebQueueIndexer::indexfiles no db??\n"); LOGERR("WebQueueIndexer::indexfiles no db??\n");
return false; return false;
} }
for (list<string>::iterator it = files.begin(); it != files.end();) { for (auto it = files.begin(); it != files.end();) {
if (it->empty()) {//?? if (it->empty()) {//??
it++; continue; it++; continue;
} }
string father = path_getfather(*it); string father = path_getfather(*it);
if (father.compare(m_queuedir)) { if (father.compare(m_queuedir)) {
LOGDEB("WebQueueIndexer::indexfiles: skipping [" << *it << "] (nq)\n"); LOGDEB("WebQueueIndexer::indexfiles: skipping ["<<*it << "] (nq)\n");
it++; continue; it++; continue;
} }
// Pb: we are often called with the dot file, before the // Pb: we are often called with the dot file, before the
@ -342,12 +341,12 @@ bool WebQueueIndexer::indexFiles(list<string>& files)
if (fn.empty() || fn.at(0) == '.') { if (fn.empty() || fn.at(0) == '.') {
it++; continue; it++; continue;
} }
struct stat st; struct PathStat st;
if (path_fileprops(*it, &st) != 0) { if (path_fileprops(*it, &st) != 0) {
LOGERR("WebQueueIndexer::indexfiles: cant stat [" << *it << "]\n"); LOGERR("WebQueueIndexer::indexfiles: cant stat [" << *it << "]\n");
it++; continue; it++; continue;
} }
if (!S_ISREG(st.st_mode)) { if (st.pst_type != PathStat::PST_REGULAR) {
LOGDEB("WebQueueIndexer::indexfiles: skipping [" << *it << LOGDEB("WebQueueIndexer::indexfiles: skipping [" << *it <<
"] (nr)\n"); "] (nr)\n");
it++; continue; it++; continue;
@ -364,14 +363,15 @@ bool WebQueueIndexer::indexFiles(list<string>& files)
FsTreeWalker::Status FsTreeWalker::Status
WebQueueIndexer::processone(const string &path, WebQueueIndexer::processone(const string &path,
const struct stat *stp, const struct PathStat *stp,
FsTreeWalker::CbFlag flg) FsTreeWalker::CbFlag flg)
{ {
if (!m_db) //?? if (!m_db) //??
return FsTreeWalker::FtwError; return FsTreeWalker::FtwError;
bool dounlink = false; bool dounlink = false;
string ascdate;
if (flg != FsTreeWalker::FtwRegular) if (flg != FsTreeWalker::FtwRegular)
return FsTreeWalker::FtwOk; return FsTreeWalker::FtwOk;
@ -392,8 +392,7 @@ WebQueueIndexer::processone(const string &path,
make_udi(udipath, cstr_null, udi); make_udi(udipath, cstr_null, udi);
LOGDEB("WebQueueIndexer: prc1: udi [" << udi << "]\n"); LOGDEB("WebQueueIndexer: prc1: udi [" << udi << "]\n");
char ascdate[30]; ascdate = lltodecstr(stp->pst_mtime);
sprintf(ascdate, "%ld", long(stp->st_mtime));
if (!stringlowercmp("bookmark", dotdoc.meta[Rcl::Doc::keybght])) { if (!stringlowercmp("bookmark", dotdoc.meta[Rcl::Doc::keybght])) {
// For bookmarks, we just index the doc that was built from the // For bookmarks, we just index the doc that was built from the
@ -401,7 +400,7 @@ WebQueueIndexer::processone(const string &path,
if (dotdoc.fmtime.empty()) if (dotdoc.fmtime.empty())
dotdoc.fmtime = ascdate; dotdoc.fmtime = ascdate;
dotdoc.pcbytes = lltodecstr(stp->st_size); dotdoc.pcbytes = lltodecstr(stp->pst_size);
// Document signature for up to date checks: none. // Document signature for up to date checks: none.
dotdoc.sig.clear(); dotdoc.sig.clear();
@ -439,7 +438,7 @@ WebQueueIndexer::processone(const string &path,
doc.fmtime = ascdate; doc.fmtime = ascdate;
dotdoc.fmtime = doc.fmtime; dotdoc.fmtime = doc.fmtime;
doc.pcbytes = lltodecstr(stp->st_size); doc.pcbytes = lltodecstr(stp->pst_size);
// Document signature for up to date checks: none. // Document signature for up to date checks: none.
doc.sig.clear(); doc.sig.clear();
doc.url = dotdoc.url; doc.url = dotdoc.url;

12
src/index/webqueue.h
View File

@ -50,7 +50,7 @@ public:
/** Called when we fstreewalk the queue dir */ /** Called when we fstreewalk the queue dir */
FsTreeWalker::Status FsTreeWalker::Status
processone(const string &, const struct stat *, FsTreeWalker::CbFlag); processone(const std::string &, const struct PathStat *, FsTreeWalker::CbFlag);
/** Index a list of files. No db cleaning or stemdb updating. /** Index a list of files. No db cleaning or stemdb updating.
* Used by the real time monitor */ * Used by the real time monitor */
@ -62,18 +62,18 @@ public:
/** Called when indexing data from the cache, and from internfile for /** Called when indexing data from the cache, and from internfile for
* search result preview */ * search result preview */
bool getFromCache(const string& udi, Rcl::Doc &doc, string& data, bool getFromCache(const std::string& udi, Rcl::Doc &doc, std::string& data,
string *hittype = 0); std::string *hittype = 0);
private: private:
RclConfig *m_config; RclConfig *m_config;
Rcl::Db *m_db; Rcl::Db *m_db;
WebStore *m_cache; WebStore *m_cache;
string m_queuedir; std::string m_queuedir;
DbIxStatusUpdater *m_updater; DbIxStatusUpdater *m_updater;
bool m_nocacheindex; bool m_nocacheindex;
bool indexFromCache(const string& udi); bool indexFromCache(const std::string& udi);
void updstatus(const string& udi); void updstatus(const std::string& udi);
}; };
#endif /* _webqueue_h_included_ */ #endif /* _webqueue_h_included_ */

21
src/internfile/internfile.cpp
View File

@ -22,7 +22,6 @@
#include <stdint.h> #include <stdint.h>
#include "safefcntl.h" #include "safefcntl.h"
#include <sys/types.h> #include <sys/types.h>
#include "safesysstat.h"
#include "safeunistd.h" #include "safeunistd.h"
#include <string> #include <string>
@ -117,7 +116,7 @@ bool FileInterner::ipathContains(const string& parent, const string& child)
// Empty handler on return says that we're in error, this will be // Empty handler on return says that we're in error, this will be
// processed by the first call to internfile(). // processed by the first call to internfile().
// Split into "constructor calls init()" to allow use from other constructor // Split into "constructor calls init()" to allow use from other constructor
FileInterner::FileInterner(const string &fn, const struct stat *stp, FileInterner::FileInterner(const string &fn, const struct PathStat *stp,
RclConfig *cnf, int flags, const string *imime) RclConfig *cnf, int flags, const string *imime)
{ {
LOGDEB0("FileInterner::FileInterner(fn=" << fn << ")\n"); LOGDEB0("FileInterner::FileInterner(fn=" << fn << ")\n");
@ -137,8 +136,8 @@ FileInterner::FileInterner(const string &fn, const struct stat *stp,
// used to not be the case, and was changed because this was the // used to not be the case, and was changed because this was the
// simplest way to solve the retry issues (simpler than changing the // simplest way to solve the retry issues (simpler than changing the
// caller in e.g. fsindexer). // caller in e.g. fsindexer).
void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf, void FileInterner::init(const string &f, const struct PathStat *stp,
int flags, const string *imime) RclConfig *cnf, int flags, const string *imime)
{ {
if (f.empty()) { if (f.empty()) {
LOGERR("FileInterner::init: empty file name!\n"); LOGERR("FileInterner::init: empty file name!\n");
@ -185,7 +184,7 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
l_mime = *imime; l_mime = *imime;
} }
int64_t docsize = stp->st_size; int64_t docsize = stp->pst_size;
if (!l_mime.empty()) { if (!l_mime.empty()) {
// Has mime: check for a compressed file. If so, create a // Has mime: check for a compressed file. If so, create a
@ -196,7 +195,7 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
// Check for compressed size limit // Check for compressed size limit
int maxkbs = -1; int maxkbs = -1;
if (!m_cfg->getConfParam("compressedfilemaxkbs", &maxkbs) || if (!m_cfg->getConfParam("compressedfilemaxkbs", &maxkbs) ||
maxkbs < 0 || !stp || int(stp->st_size / 1024) < maxkbs) { maxkbs < 0 || !stp || int(stp->pst_size / 1024) < maxkbs) {
if (!m_uncomp->uncompressfile(m_fn, ucmd, m_tfile)) { if (!m_uncomp->uncompressfile(m_fn, ucmd, m_tfile)) {
m_ok = true; m_ok = true;
return; return;
@ -204,14 +203,14 @@ void FileInterner::init(const string &f, const struct stat *stp, RclConfig *cnf,
LOGDEB1("FileInterner:: after ucomp: tfile " << m_tfile <<"\n"); LOGDEB1("FileInterner:: after ucomp: tfile " << m_tfile <<"\n");
m_fn = m_tfile; m_fn = m_tfile;
// Stat the uncompressed file, mainly to get the size // Stat the uncompressed file, mainly to get the size
struct stat ucstat; struct PathStat ucstat;
if (path_fileprops(m_fn, &ucstat) != 0) { if (path_fileprops(m_fn, &ucstat) != 0) {
LOGERR("FileInterner: can't stat the uncompressed file[" << LOGERR("FileInterner: can't stat the uncompressed file[" <<
m_fn << "] errno " << errno << "\n"); m_fn << "] errno " << errno << "\n");
m_ok = true; m_ok = true;
return; return;
} else { } else {
docsize = ucstat.st_size; docsize = ucstat.pst_size;
} }
l_mime = mimetype(m_fn, &ucstat, m_cfg, usfci); l_mime = mimetype(m_fn, &ucstat, m_cfg, usfci);
if (l_mime.empty() && imime) if (l_mime.empty() && imime)
@ -1096,7 +1095,7 @@ bool FileInterner::interntofile(TempFile& otemp, const string& tofile,
bool FileInterner::isCompressed(const string& fn, RclConfig *cnf) bool FileInterner::isCompressed(const string& fn, RclConfig *cnf)
{ {
LOGDEB("FileInterner::isCompressed: [" << fn << "]\n"); LOGDEB("FileInterner::isCompressed: [" << fn << "]\n");
struct stat st; struct PathStat st;
if (path_fileprops(fn, &st) < 0) { if (path_fileprops(fn, &st) < 0) {
LOGERR("FileInterner::isCompressed: can't stat [" << fn << "]\n"); LOGERR("FileInterner::isCompressed: can't stat [" << fn << "]\n");
return false; return false;
@ -1120,7 +1119,7 @@ bool FileInterner::maybeUncompressToTemp(TempFile& temp, const string& fn,
RclConfig *cnf, const Rcl::Doc& doc) RclConfig *cnf, const Rcl::Doc& doc)
{ {
LOGDEB("FileInterner::maybeUncompressToTemp: [" << fn << "]\n"); LOGDEB("FileInterner::maybeUncompressToTemp: [" << fn << "]\n");
struct stat st; struct PathStat st;
if (path_fileprops(fn.c_str(), &st) < 0) { if (path_fileprops(fn.c_str(), &st) < 0) {
LOGERR("FileInterner::maybeUncompressToTemp: can't stat [" <<fn<<"]\n"); LOGERR("FileInterner::maybeUncompressToTemp: can't stat [" <<fn<<"]\n");
return false; return false;
@ -1139,7 +1138,7 @@ bool FileInterner::maybeUncompressToTemp(TempFile& temp, const string& fn,
// Check for compressed size limit // Check for compressed size limit
int maxkbs = -1; int maxkbs = -1;
if (cnf->getConfParam("compressedfilemaxkbs", &maxkbs) && if (cnf->getConfParam("compressedfilemaxkbs", &maxkbs) &&
maxkbs >= 0 && int(st.st_size / 1024) > maxkbs) { maxkbs >= 0 && int(st.pst_size / 1024) > maxkbs) {
LOGINFO("FileInterner:: " << fn << " over size limit " << maxkbs << LOGINFO("FileInterner:: " << fn << " over size limit " << maxkbs <<
" kbs\n"); " kbs\n");
return false; return false;

6
src/internfile/internfile.h
View File

@ -37,7 +37,7 @@ class Doc;
} }
class Uncomp; class Uncomp;
struct stat; struct PathStat;
/** Storage for missing helper program info. We want to keep this out of the /** Storage for missing helper program info. We want to keep this out of the
* FileInterner class, because the data will typically be accumulated by several * FileInterner class, because the data will typically be accumulated by several
@ -114,7 +114,7 @@ public:
* @param mtype mime type if known. For a compressed file this is the * @param mtype mime type if known. For a compressed file this is the
* mime type for the uncompressed version. * mime type for the uncompressed version.
*/ */
FileInterner(const string &fn, const struct stat *stp, FileInterner(const string &fn, const struct PathStat *stp,
RclConfig *cnf, int flags, const string *mtype = 0); RclConfig *cnf, int flags, const string *mtype = 0);
/** /**
@ -290,7 +290,7 @@ private:
bool m_direct; // External app did the extraction bool m_direct; // External app did the extraction
// Pseudo-constructors // Pseudo-constructors
void init(const string &fn, const struct stat *stp, void init(const string &fn, const struct PathStat *stp,
RclConfig *cnf, int flags, const string *mtype = 0); RclConfig *cnf, int flags, const string *mtype = 0);
void init(const string &data, RclConfig *cnf, int flags, void init(const string &data, RclConfig *cnf, int flags,
const string& mtype); const string& mtype);

BIN
src/qtgui/i18n/recoll_ko.qm
View File

Binary file not shown.

9
src/qtgui/i18n/recoll_ko.ts
View File

@ -247,7 +247,7 @@
</message> </message>
<message> <message>
<source>Index flush megabytes interval</source> <source>Index flush megabytes interval</source>
<translation>&lt;p&gt; 이것은 기본적으로 모든 분음 부호를 제거하고 정식 분해를 수행하는 unac 메커니즘에 대한 예외를 설정합니다. 언어에 따라 일부 문자의 강조를 무시, 추가, 분해를 지정할 수 있습니다(예 : 합자의 경우. 공백으로 구분된 각 항목에서 첫 번째 문자는 원본이고 나머지는 번역입니다).</translation> <translation>색인 정비 간격(MB)</translation>
</message> </message>
<message> <message>
<source>This value adjust the amount of data which is indexed between flushes to disk.&lt;br&gt;This helps control the indexer memory usage. Default 10MB </source> <source>This value adjust the amount of data which is indexed between flushes to disk.&lt;br&gt;This helps control the indexer memory usage. Default 10MB </source>
@ -1219,7 +1219,7 @@ Do you want to start the preferences dialog ?</source>
</message> </message>
<message> <message>
<source>Query in progress.&lt;br&gt;Due to limitations of the indexing library,&lt;br&gt;cancelling will exit the program</source> <source>Query in progress.&lt;br&gt;Due to limitations of the indexing library,&lt;br&gt;cancelling will exit the program</source>
<translation>검색어 요청이 진행중입니다.&lt;br&gt; 색인 작업 보관용량의 제한으로 인해&lt;br&gt; 프로그램은 취소되고 종료될 것입니다.</translation> <translation>검색어 요청이 진행중입니다.&lt;br&gt;취소되면 색인 작업 보관용량의 제한 때문에&lt;br&gt;프로그램이 종료될 것입니다.</translation>
</message> </message>
<message> <message>
<source>Error</source> <source>Error</source>
@ -2407,7 +2407,7 @@ Use &lt;b&gt;Show Query&lt;/b&gt; link when in doubt about result and see manual
</message> </message>
<message> <message>
<source>terms may be missing. Try using a longer root.</source> <source>terms may be missing. Try using a longer root.</source>
<translation>용어가 아마 누락된 것 같습니다. 더 긴 root를 사용하여 시도하십시오.</translation> <translation>용어가 아마 누락된 것 같습니다. 더 긴 어근을 사용하여 시도하십시오.</translation>
</message> </message>
<message> <message>
<source>Show index statistics</source> <source>Show index statistics</source>
@ -2864,8 +2864,7 @@ This will help searching very big text files (ie: log files).</source>
</message> </message>
<message> <message>
<source>Index flush megabytes interval</source> <source>Index flush megabytes interval</source>
<translatorcomment>격증:flush</translatorcomment> <translation type="vanished">색인 정비 간격(MB)</translation>
<translation type="vanished">&lt;p&gt; 이것은 기본적으로 모든 분음 부호를 제거하고 정식 분해를 수행하는 unac 메커니즘에 대한 예외를 설정합니다. 언어에 따라 일부 문자의 강조를 무시, 추가, 분해를 지정할 수 있습니다(예 : 합자의 경우. 공백으로 구분된 각 항목에서 첫 번째 문자는 원본이고 나머지는 번역입니다).</translation>
</message> </message>
<message> <message>
<source>This value adjust the amount of data which is indexed between flushes to disk.&lt;br&gt;This helps control the indexer memory usage. Default 10MB </source> <source>This value adjust the amount of data which is indexed between flushes to disk.&lt;br&gt;This helps control the indexer memory usage. Default 10MB </source>

5
src/testmains/Makefile.am
View File

@ -38,11 +38,14 @@ AM_CPPFLAGS = -Wall -Wno-unused -std=c++11 \
$(DEFS) $(DEFS)
noinst_PROGRAMS = textsplit utf8iter fstreewalk rclconfig hldata unac mbox \ noinst_PROGRAMS = textsplit utf8iter fstreewalk rclconfig hldata unac mbox \
circache wipedir circache wipedir mimetype
textsplit_SOURCES = trtextsplit.cpp textsplit_SOURCES = trtextsplit.cpp
textsplit_LDADD = ../librecoll.la textsplit_LDADD = ../librecoll.la
mimetype_SOURCES = trmimetype.cpp
mimetype_LDADD = ../librecoll.la
utf8iter_SOURCES = trutf8iter.cpp utf8iter_SOURCES = trutf8iter.cpp
utf8iter_LDADD = ../librecoll.la utf8iter_LDADD = ../librecoll.la

63
src/testmains/trfstreewalk.cpp
View File

@ -24,6 +24,7 @@
#include "rclinit.h" #include "rclinit.h"
#include "rclconfig.h" #include "rclconfig.h"
#include "pathut.h"
using namespace std; using namespace std;
@ -47,25 +48,25 @@ static int op_flags;
class myCB : public FsTreeWalkerCB { class myCB : public FsTreeWalkerCB {
public: public:
FsTreeWalker::Status processone(const string &path, FsTreeWalker::Status processone(const string &path,
const struct stat *st, const struct PathStat *st,
FsTreeWalker::CbFlag flg) { FsTreeWalker::CbFlag flg) {
if (flg == FsTreeWalker::FtwDirEnter) { if (flg == FsTreeWalker::FtwDirEnter) {
if (op_flags & OPT_r) { if (op_flags & OPT_r) {
cout << path << endl;
} else {
if (!(op_flags&OPT_s)) {
cout << "[Entering " << path << "]" << endl;
}
}
} else if (flg == FsTreeWalker::FtwDirReturn) {
if (!(op_flags&OPT_s)) {
cout << "[Returning to " << path << "]" << endl;
}
} else if (flg == FsTreeWalker::FtwRegular) {
cout << path << endl; cout << path << endl;
} else {
if (!(op_flags&OPT_s)) {
cout << "[Entering " << path << "]" << endl;
}
} }
return FsTreeWalker::FtwOk; } else if (flg == FsTreeWalker::FtwDirReturn) {
if (!(op_flags&OPT_s)) {
cout << "[Returning to " << path << "]" << endl;
}
} else if (flg == FsTreeWalker::FtwRegular) {
cout << path << endl;
} }
return FsTreeWalker::FtwOk;
}
}; };
static const char *thisprog; static const char *thisprog;
@ -89,22 +90,22 @@ static const char *thisprog;
// real 17m10.585s user 0m4.532s sys 0m35.033s // real 17m10.585s user 0m4.532s sys 0m35.033s
static char usage [] = static char usage [] =
"trfstreewalk [-p pattern] [-P ignpath] [-r] [-c] [-L] topdir\n" "trfstreewalk [-p pattern] [-P ignpath] [-r] [-c] [-L] topdir\n"
" -D : skip dotfiles\n" " -D : skip dotfiles\n"
" -L : follow symbolic links\n" " -L : follow symbolic links\n"
" -M <depth>: limit depth (works with -b/m/d)\n" " -M <depth>: limit depth (works with -b/m/d)\n"
" -P <pattern> : add skippedPaths entry\n" " -P <pattern> : add skippedPaths entry\n"
" -p <pattern> : add skippedNames entry\n" " -p <pattern> : add skippedNames entry\n"
" -b : use breadth first walk\n" " -b : use breadth first walk\n"
" -c : no path canonification\n" " -c : no path canonification\n"
" -d : use almost depth first (dir files, then subdirs)\n" " -d : use almost depth first (dir files, then subdirs)\n"
" -k : like du\n" " -k : like du\n"
" -m : use breadth up to 4 deep then switch to -d\n" " -m : use breadth up to 4 deep then switch to -d\n"
" -r : norecurse\n" " -r : norecurse\n"
" -s : don't print dir change info\n" " -s : don't print dir change info\n"
" -w : unset default FNM_PATHNAME when using fnmatch() to match skipped paths\n" " -w : unset default FNM_PATHNAME when using fnmatch() to match skipped paths\n"
" -y <pattern> : add onlyNames entry\n" " -y <pattern> : add onlyNames entry\n"
; ;
static void static void
Usage(void) Usage(void)
{ {

38
src/testmains/trmimetype.cpp Normal file
View File

@ -0,0 +1,38 @@
#include "mimetype.h"
#include <stdio.h>
#include <cstdlib>
#include <iostream>
#include "log.h"
#include "rclconfig.h"
#include "rclinit.h"
#include "pathut.h"
using namespace std;
int main(int argc, const char **argv)
{
string reason;
RclConfig *config = recollinit(0, 0, 0, reason);
if (config == 0 || !config->ok()) {
string str = "Configuration problem: ";
str += reason;
fprintf(stderr, "%s\n", str.c_str());
exit(1);
}
while (--argc > 0) {
string filename = *++argv;
struct PathStat st;
if (path_fileprops(filename, &st)) {
fprintf(stderr, "Can't stat %s\n", filename.c_str());
continue;
}
cout << filename << " -> " <<
mimetype(filename, &st, config, true) << endl;
}
return 0;
}

7
src/utils/appformime.cpp
View File

@ -37,12 +37,13 @@ public:
{ {
} }
virtual FsTreeWalker::Status virtual FsTreeWalker::Status
processone(const string &, const struct stat *, FsTreeWalker::CbFlag); processone(const string &, const struct PathStat *, FsTreeWalker::CbFlag);
DesktopDb::AppMap *m_appdefs; DesktopDb::AppMap *m_appdefs;
}; };
FsTreeWalker::Status FstCb::processone(const string& fn, const struct stat *, struct PathStat;
FsTreeWalker::CbFlag flg) FsTreeWalker::Status FstCb::processone(
const string& fn, const struct PathStat*, FsTreeWalker::CbFlag flg)
{ {
if (flg != FsTreeWalker::FtwRegular) if (flg != FsTreeWalker::FtwRegular)
return FsTreeWalker::FtwOk; return FsTreeWalker::FtwOk;

27
src/utils/fstreewalk.cpp
View File

@ -21,7 +21,6 @@
#include <dirent.h> #include <dirent.h>
#include <errno.h> #include <errno.h>
#include <fnmatch.h> #include <fnmatch.h>
#include "safesysstat.h"
#include <cstring> #include <cstring>
#include <algorithm> #include <algorithm>
@ -228,8 +227,7 @@ static inline int slashcount(const string& p)
return n; return n;
} }
FsTreeWalker::Status FsTreeWalker::walk(const string& _top, FsTreeWalker::Status FsTreeWalker::walk(const string& _top, FsTreeWalkerCB& cb)
FsTreeWalkerCB& cb)
{ {
string top = (data->options & FtwNoCanon) ? _top : path_canon(_top); string top = (data->options & FtwNoCanon) ? _top : path_canon(_top);
@ -238,7 +236,7 @@ FsTreeWalker::Status FsTreeWalker::walk(const string& _top,
} }
data->basedepth = slashcount(top); // Only used for breadthxx data->basedepth = slashcount(top); // Only used for breadthxx
struct stat st; struct PathStat st;
// We always follow symlinks at this point. Makes more sense. // We always follow symlinks at this point. Makes more sense.
if (path_fileprops(top, &st) == -1) { if (path_fileprops(top, &st) == -1) {
// Note that we do not return an error if the stat call // Note that we do not return an error if the stat call
@ -343,19 +341,19 @@ FsTreeWalker::Status FsTreeWalker::walk(const string& _top,
// This means that we always go into the top 'walk()' parameter if it is a // This means that we always go into the top 'walk()' parameter if it is a
// directory, even if norecurse is set. Bug or Feature ? // directory, even if norecurse is set. Bug or Feature ?
FsTreeWalker::Status FsTreeWalker::iwalk(const string &top, FsTreeWalker::Status FsTreeWalker::iwalk(const string &top,
struct stat *stp, struct PathStat *stp,
FsTreeWalkerCB& cb) FsTreeWalkerCB& cb)
{ {
Status status = FtwOk; Status status = FtwOk;
bool nullpush = false; bool nullpush = false;
// Tell user to process the top entry itself // Tell user to process the top entry itself
if (S_ISDIR(stp->st_mode)) { if (stp->pst_type == PathStat::PST_DIR) {
if ((status = cb.processone(top, stp, FtwDirEnter)) & if ((status = cb.processone(top, stp, FtwDirEnter)) &
(FtwStop|FtwError)) { (FtwStop|FtwError)) {
return status; return status;
} }
} else if (S_ISREG(stp->st_mode)) { } else if (stp->pst_type == PathStat::PST_REGULAR) {
return cb.processone(top, stp, FtwRegular); return cb.processone(top, stp, FtwRegular);
} else { } else {
return status; return status;
@ -377,7 +375,7 @@ FsTreeWalker::Status FsTreeWalker::iwalk(const string &top,
// For now, we'll ignore the "other kind of cycle" part and only monitor // For now, we'll ignore the "other kind of cycle" part and only monitor
// this is FtwFollow is set // this is FtwFollow is set
if (data->options & FtwFollow) { if (data->options & FtwFollow) {
DirId dirid(stp->st_dev, stp->st_ino); DirId dirid(stp->pst_dev, stp->pst_ino);
if (data->donedirs.find(dirid) != data->donedirs.end()) { if (data->donedirs.find(dirid) != data->donedirs.end()) {
LOGINFO("Not processing [" << top << LOGINFO("Not processing [" << top <<
"] (already seen as other path)\n"); "] (already seen as other path)\n");
@ -420,7 +418,7 @@ FsTreeWalker::Status FsTreeWalker::iwalk(const string &top,
struct DIRENT *ent; struct DIRENT *ent;
while (errno = 0, ((ent = READDIR(d)) != 0)) { while (errno = 0, ((ent = READDIR(d)) != 0)) {
string fn; string fn;
struct stat st; struct PathStat st;
#ifdef _WIN32 #ifdef _WIN32
string sdname; string sdname;
if (!wchartoutf8(ent->d_name, sdname)) { if (!wchartoutf8(ent->d_name, sdname)) {
@ -468,7 +466,7 @@ FsTreeWalker::Status FsTreeWalker::iwalk(const string &top,
continue; continue;
} }
if (S_ISDIR(st.st_mode)) { if (st.pst_type == PathStat::PST_DIR) {
if (!o_nowalkfn.empty() && path_exists(path_cat(fn, o_nowalkfn))) { if (!o_nowalkfn.empty() && path_exists(path_cat(fn, o_nowalkfn))) {
continue; continue;
} }
@ -498,7 +496,8 @@ FsTreeWalker::Status FsTreeWalker::iwalk(const string &top,
if ((status = cb.processone(top, &st, FtwDirReturn)) if ((status = cb.processone(top, &st, FtwDirReturn))
& (FtwStop|FtwError)) & (FtwStop|FtwError))
goto out; goto out;
} else if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { } else if (st.pst_type == PathStat::PST_REGULAR ||
st.pst_type == PathStat::PST_SYMLINK) {
// Filtering patterns match ? // Filtering patterns match ?
if (!data->onlyNames.empty()) { if (!data->onlyNames.empty()) {
if (!inOnlyNames(dname)) if (!inOnlyNames(dname))
@ -536,14 +535,14 @@ int64_t fsTreeBytes(const string& topdir)
class bytesCB : public FsTreeWalkerCB { class bytesCB : public FsTreeWalkerCB {
public: public:
FsTreeWalker::Status processone(const string &path, FsTreeWalker::Status processone(const string &path,
const struct stat *st, const struct PathStat *st,
FsTreeWalker::CbFlag flg) { FsTreeWalker::CbFlag flg) {
if (flg == FsTreeWalker::FtwDirEnter || if (flg == FsTreeWalker::FtwDirEnter ||
flg == FsTreeWalker::FtwRegular) { flg == FsTreeWalker::FtwRegular) {
#ifdef _WIN32 #ifdef _WIN32
totalbytes += st->st_size; totalbytes += st->pst_size;
#else #else
totalbytes += st->st_blocks * 512; totalbytes += st->pst_blocks * 512;
#endif #endif
} }
return FsTreeWalker::FtwOk; return FsTreeWalker::FtwOk;

42
src/utils/fstreewalk.h
View File

@ -19,13 +19,10 @@
#include <string> #include <string>
#include <vector> #include <vector>
#ifndef NO_NAMESPACES
using std::string; struct PathStat;
using std::vector;
#endif
class FsTreeWalkerCB; class FsTreeWalkerCB;
struct stat;
/** /**
* Class implementing a unix directory recursive walk. * Class implementing a unix directory recursive walk.
@ -44,8 +41,7 @@ class FsTreeWalker {
// to not use the flag, which can be set from rclconfig by adding // to not use the flag, which can be set from rclconfig by adding
// a value to the config file (skippedPathsNoFnmPathname) // a value to the config file (skippedPathsNoFnmPathname)
static bool o_useFnmPathname; static bool o_useFnmPathname;
static void setNoFnmPathname() static void setNoFnmPathname() {
{
o_useFnmPathname = false; o_useFnmPathname = false;
} }
@ -53,8 +49,7 @@ class FsTreeWalker {
// directories as if they were in skippedPaths) if the file exists // directories as if they were in skippedPaths) if the file exists
// inside the directory. // inside the directory.
static std::string o_nowalkfn; static std::string o_nowalkfn;
static void setNoWalkFn(const std::string& nowalkfn) static void setNoWalkFn(const std::string& nowalkfn) {
{
o_nowalkfn = nowalkfn; o_nowalkfn = nowalkfn;
} }
@ -97,34 +92,35 @@ class FsTreeWalker {
* @param cb the function object that will be called back for every * @param cb the function object that will be called back for every
* file-system object (called both at entry and exit for directories). * file-system object (called both at entry and exit for directories).
*/ */
Status walk(const string &dir, FsTreeWalkerCB& cb); Status walk(const std::string &dir, FsTreeWalkerCB& cb);
/** Get explanation for error */ /** Get explanation for error */
string getReason(); std::string getReason();
int getErrCnt(); int getErrCnt();
/** /**
* Add a pattern (file or dir) to be ignored (ie: #* , *~) * Add a pattern (file or dir) to be ignored (ie: #* , *~)
*/ */
bool addSkippedName(const string &pattern); bool addSkippedName(const std::string &pattern);
/** Set the ignored patterns set */ /** Set the ignored patterns set */
bool setSkippedNames(const vector<string> &patterns); bool setSkippedNames(const std::vector<std::string> &patterns);
/** Set the exclusive patterns set */ /** Set the exclusive patterns set */
bool setOnlyNames(const vector<string> &patterns); bool setOnlyNames(const std::vector<std::string> &patterns);
/** Same for skipped paths: this are paths, not names, under which we /** Same for skipped paths: this are paths, not names, under which we
do not descend (ie: /home/me/.recoll) */ do not descend (ie: /home/me/.recoll) */
bool addSkippedPath(const string &path); bool addSkippedPath(const std::string &path);
/** Set the ignored paths list */ /** Set the ignored paths list */
bool setSkippedPaths(const vector<string> &patterns); bool setSkippedPaths(const std::vector<std::string> &patterns);
/** Test if path/name should be skipped. This can be used independently of /** Test if path/name should be skipped. This can be used independently of
* an actual tree walk */ * an actual tree walk */
bool inSkippedPaths(const string& path, bool ckparents = false); bool inSkippedPaths(const std::string& path, bool ckparents = false);
bool inSkippedNames(const string& name); bool inSkippedNames(const std::string& name);
bool inOnlyNames(const string& name); bool inOnlyNames(const std::string& name);
private: private:
Status iwalk(const string &dir, struct stat *stp, FsTreeWalkerCB& cb); Status iwalk(const std::string &dir, struct PathStat *stp,
FsTreeWalkerCB& cb);
class Internal; class Internal;
Internal *data; Internal *data;
}; };
@ -134,11 +130,11 @@ class FsTreeWalkerCB {
virtual ~FsTreeWalkerCB() {} virtual ~FsTreeWalkerCB() {}
// Only st_mtime, st_ctime, st_size, st_mode (filetype bits: dir/reg/lnk), // Only st_mtime, st_ctime, st_size, st_mode (filetype bits: dir/reg/lnk),
virtual FsTreeWalker::Status virtual FsTreeWalker::Status
processone(const string &, const struct stat *, FsTreeWalker::CbFlag) processone(const std::string&, const struct PathStat *,
= 0; FsTreeWalker::CbFlag) = 0;
}; };
// Utility function. Somewhat like du. // Utility function. Somewhat like du.
int64_t fsTreeBytes(const string& topdir); int64_t fsTreeBytes(const std::string& topdir);
#endif /* _FSTREEWALK_H_INCLUDED_ */ #endif /* _FSTREEWALK_H_INCLUDED_ */

49
src/utils/pathut.cpp
View File

@ -684,11 +684,12 @@ bool path_makepath(const string& ipath, int mode)
return true; return true;
} }
bool path_isdir(const string& path) bool path_isdir(const string& path, bool follow)
{ {
struct STATBUF st; struct STATBUF st;
SYSPATH(path, syspath); SYSPATH(path, syspath);
if (LSTAT(syspath, &st) < 0) { int ret = follow ? STAT(syspath, &st) : LSTAT(syspath, &st);
if (ret < 0) {
return false; return false;
} }
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
@ -697,6 +698,20 @@ bool path_isdir(const string& path)
return false; return false;
} }
bool path_isfile(const string& path, bool follow)
{
struct STATBUF st;
SYSPATH(path, syspath);
int ret = follow ? STAT(syspath, &st) : LSTAT(syspath, &st);
if (ret < 0) {
return false;
}
if (S_ISREG(st.st_mode)) {
return true;
}
return false;
}
long long path_filesize(const string& path) long long path_filesize(const string& path)
{ {
struct STATBUF st; struct STATBUF st;
@ -707,30 +722,36 @@ long long path_filesize(const string& path)
return (long long)st.st_size; return (long long)st.st_size;
} }
int path_fileprops(const std::string path, struct stat *stp, bool follow) int path_fileprops(const std::string path, struct PathStat *stp, bool follow)
{ {
if (!stp) { if (nullptr == stp) {
return -1; return -1;
} }
memset(stp, 0, sizeof(struct stat)); memset(stp, 0, sizeof(struct PathStat));
struct STATBUF mst; struct STATBUF mst;
SYSPATH(path, syspath); SYSPATH(path, syspath);
int ret = follow ? STAT(syspath, &mst) : LSTAT(syspath, &mst); int ret = follow ? STAT(syspath, &mst) : LSTAT(syspath, &mst);
if (ret != 0) { if (ret != 0) {
return ret; return ret;
} }
stp->st_size = mst.st_size; stp->pst_size = mst.st_size;
stp->st_mode = mst.st_mode; stp->pst_mode = mst.st_mode;
stp->st_mtime = mst.st_mtime; stp->pst_mtime = mst.st_mtime;
#ifdef _WIN32 #ifdef _WIN32
stp->st_ctime = mst.st_mtime; stp->pst_ctime = mst.st_mtime;
#else #else
stp->st_ino = mst.st_ino; stp->pst_ino = mst.st_ino;
stp->st_dev = mst.st_dev; stp->pst_dev = mst.st_dev;
stp->st_ctime = mst.st_ctime; stp->pst_ctime = mst.st_ctime;
stp->st_blocks = mst.st_blocks; stp->pst_blocks = mst.st_blocks;
stp->st_blksize = mst.st_blksize; stp->pst_blksize = mst.st_blksize;
#endif #endif
switch (mst.st_mode & S_IFMT) {
case S_IFDIR: stp->pst_type = PathStat::PST_DIR;break;
case S_IFLNK: stp->pst_type = PathStat::PST_SYMLINK;break;
case S_IFREG: stp->pst_type = PathStat::PST_REGULAR;break;
default: stp->pst_type = PathStat::PST_OTHER;break;
}
return 0; return 0;
} }

23
src/utils/pathut.h
View File

@ -20,6 +20,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include <set> #include <set>
#include <cstdint>
// Must be called in main thread before starting other threads // Must be called in main thread before starting other threads
extern void pathut_init_mt(); extern void pathut_init_mt();
@ -66,7 +67,9 @@ extern std::string url_parentfolder(const std::string& url);
extern std::string url_gpath(const std::string& url); extern std::string url_gpath(const std::string& url);
/// Stat parameter and check if it's a directory /// Stat parameter and check if it's a directory
extern bool path_isdir(const std::string& path); extern bool path_isdir(const std::string& path, bool follow = false);
/// Stat parameter and check if it's a regular file
extern bool path_isfile(const std::string& path, bool follow = false);
/// Retrieve file size /// Retrieve file size
extern long long path_filesize(const std::string& path); extern long long path_filesize(const std::string& path);
@ -79,8 +82,19 @@ extern long long path_filesize(const std::string& path);
/// all systems. st_dev and st_ino are set for special posix usage. /// all systems. st_dev and st_ino are set for special posix usage.
/// The rest is zeroed. /// The rest is zeroed.
/// @ret 0 for success /// @ret 0 for success
struct stat; struct PathStat {
extern int path_fileprops(const std::string path, struct stat *stp, enum PstType {PST_REGULAR, PST_SYMLINK, PST_DIR, PST_OTHER};
PstType pst_type;
int64_t pst_size;
uint64_t pst_mode;
int64_t pst_mtime;
int64_t pst_ctime;
uint64_t pst_ino;
uint64_t pst_dev;
uint64_t pst_blocks;
uint64_t pst_blksize;
};
extern int path_fileprops(const std::string path, struct PathStat *stp,
bool follow = true); bool follow = true);
/// Check that path is traversable and last element exists /// Check that path is traversable and last element exists
@ -150,9 +164,6 @@ public:
/// Convert \ separators to / /// Convert \ separators to /
void path_slashize(std::string& s); void path_slashize(std::string& s);
void path_backslashize(std::string& s); void path_backslashize(std::string& s);
#include "safeunistd.h"
#else
#include <unistd.h>
#endif #endif
/// Lock/pid file class. This is quite close to the pidfile_xxx /// Lock/pid file class. This is quite close to the pidfile_xxx