tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

allow indexing individual files. Fix pb with preview and charsets (local defcharset ignored)

Browse Source
This commit is contained in:
dockes
2005-12-14 11:00:48 +00:00
parent 315f8cdb33
commit be485e8059
9 changed files with 329 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index/indexer.cpp
+64 -56
View File
@@ -1,10 +1,10 @@
#ifndef lint
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.19 2005-11-30 09:46:25 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: indexer.cpp,v 1.20 2005-12-14 11:00:48 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
#include <errno.h>
#include <strings.h>
#include <iostream>
@@ -33,63 +33,24 @@ using namespace std;
#define deleteZ(X) {delete X;X = 0;}
#endif
/// A class to index a list of top directories into one database.
///
/// Inherits FsTreeWalkerCB so that its processone() method is
/// called by the file-system tree walk code for each file and
/// directory, and keeps all state used while indexing a
/// directory tree.
class DbIndexer : public FsTreeWalkerCB {
FsTreeWalker walker;
RclConfig *config;
string dbdir;
list<string> *topdirs;
Rcl::Db db;
string tmpdir;
public:
/// Constructor does nothing but store parameters
DbIndexer(RclConfig *cnf, const string &dbd, list<string> *top)
: config(cnf), dbdir(dbd), topdirs(top) {}
virtual ~DbIndexer() {
// Maybe clean up temporary directory
if (tmpdir.length()) {
wipedir(tmpdir);
if (rmdir(tmpdir.c_str()) < 0) {
LOGERR(("DbIndexer::~DbIndexer: cant clear temp dir %s\n",
tmpdir.c_str()));
}
DbIndexer::~DbIndexer() {
// Maybe clean up temporary directory
if (tmpdir.length()) {
wipedir(tmpdir);
if (rmdir(tmpdir.c_str()) < 0) {
LOGERR(("DbIndexer::~DbIndexer: cannot clear temp dir %s\n",
tmpdir.c_str()));
}
}
/// Start indexing.
bool index(bool resetbefore);
/// Tree walker callback method
FsTreeWalker::Status
processone(const std::string &, const struct stat *, FsTreeWalker::CbFlag);
};
db.close();
}
/// Top level file system tree index method for updating a given database.
///
/// We create the temporary directory, open the database, then call a
/// file system walk for each top-level directory.
/// When walking is done, we create the stem databases and close the
/// main db.
bool DbIndexer::index(bool resetbefore)
bool DbIndexer::indexDb(bool resetbefore, list<string> *topdirs)
{
string tdir;
if (!init(resetbefore))
return false;
if (!maketmpdir(tmpdir)) {
LOGERR(("DbIndexer: cant create temp directory\n"));
return false;
}
if (!db.open(dbdir, resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd)) {
LOGERR(("DbIndexer::index: error opening database in %s\n",
dbdir.c_str()));
return false;
}
for (list<string>::const_iterator it = topdirs->begin();
it != topdirs->end(); it++) {
LOGDEB(("DbIndexer::index: Indexing %s into %s\n", it->c_str(),
@@ -118,7 +79,6 @@ bool DbIndexer::index(bool resetbefore)
if (walker.walk(*it, *this) != FsTreeWalker::FtwOk) {
LOGERR(("DbIndexer::index: error while indexing %s\n",
it->c_str()));
db.close();
return false;
}
}
@@ -138,6 +98,7 @@ bool DbIndexer::index(bool resetbefore)
}
}
// The close would be done in our destructor, but we want status here
if (!db.close()) {
LOGERR(("DbIndexer::index: error closing database in %s\n",
dbdir.c_str()));
@@ -146,6 +107,52 @@ bool DbIndexer::index(bool resetbefore)
return true;
}
bool DbIndexer::init(bool resetbefore)
{
if (!maketmpdir(tmpdir)) {
LOGERR(("DbIndexer: cannot create temporary directory\n"));
return false;
}
if (!db.open(dbdir, resetbefore ? Rcl::Db::DbTrunc : Rcl::Db::DbUpd)) {
LOGERR(("DbIndexer: error opening database in %s\n", dbdir.c_str()));
return false;
}
return true;
}
bool DbIndexer::indexFiles(const list<string> &filenames)
{
if (!init())
return false;
list<string>::const_iterator it;
for (it = filenames.begin(); it != filenames.end();it++) {
config->setKeyDir(path_getfather(*it));
struct stat stb;
if (stat(it->c_str(), &stb) != 0) {
LOGERR(("DbIndexer::indexFiles: stat(%s): %s", it->c_str(),
strerror(errno)));
continue;
}
if (!S_ISREG(stb.st_mode)) {
LOGERR(("DbIndexer::indexFiles: %s: not a regular file\n",
it->c_str()));
continue;
}
if (processone(*it, &stb, FsTreeWalker::FtwRegular) !=
FsTreeWalker::FtwOk) {
LOGERR(("DbIndexer::indexFiles: Database error\n"));
return false;
}
}
// The close would be done in our destructor, but we want status here
if (!db.close()) {
LOGERR(("DbIndexer::indexfiles: error closing database in %s\n",
dbdir.c_str()));
return false;
}
return true;
}
/// This method gets called for every file and directory found by the
/// tree walker.
@@ -257,6 +264,7 @@ bool ConfIndexer::index(bool resetbefore)
}
config->setKeyDir("");
// The dbmap now has dbdir as key and directory lists as values.
// Index each directory group in turn
for (dbit = dbmap.begin(); dbit != dbmap.end(); dbit++) {
//cout << dbit->first << " -> ";
@@ -265,8 +273,8 @@ bool ConfIndexer::index(bool resetbefore)
// cout << *dit << " ";
//}
//cout << endl;
dbindexer = new DbIndexer(config, dbit->first, &dbit->second);
if (!dbindexer->index(resetbefore)) {
dbindexer = new DbIndexer(config, dbit->first);
if (!dbindexer->indexDb(resetbefore, &dbit->second)) {
deleteZ(dbindexer);
return false;
}
src/index/indexer.h
+69 -17
View File
@@ -1,32 +1,84 @@
#ifndef _INDEXER_H_INCLUDED_
#define _INDEXER_H_INCLUDED_
/* @(#$Id: indexer.h,v 1.7 2005-11-30 09:46:25 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: indexer.h,v 1.8 2005-12-14 11:00:48 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string>
#include <list>
#include "rclconfig.h"
#include "fstreewalk.h"
#include "rcldb.h"
/**
* An internal class to process all directories indexed into the same database.
*/
/* Forward decl for lower level indexing object */
class DbIndexer;
/**
* The file system indexing object. Processes the configuration, then invokes
* file system walking to populate/update the database(s).
*
* Multiple top-level directories can be listed in the
* configuration. Each can be indexed to a different
* database. Directories are first grouped by database, then an
* internal class (DbIndexer) is used to process each group.
*/
The top level indexing object. Processes the configuration, then invokes
file system walking to populate/update the database(s).
Multiple top-level directories can be listed in the
configuration. Each can be indexed to a different
database. Directories are first grouped by database, then an
internal class (DbIndexer) is used to process each group.
*/
class ConfIndexer {
RclConfig *config;
DbIndexer *dbindexer; // Object to process directories for a given db
public:
enum runStatus {IndexerOk, IndexerError};
ConfIndexer(RclConfig *cnf) : config(cnf), dbindexer(0) {}
virtual ~ConfIndexer();
/** Worker function: doe the actual indexing */
bool index(bool resetbefore = false);
virtual ~ConfIndexer();
/** Worker function: doe the actual indexing */
bool index(bool resetbefore = false);
private:
RclConfig *config;
DbIndexer *dbindexer; // Object to process directories for a given db
};
/** Index things into one database
Tree indexing: we inherits FsTreeWalkerCB so that, the processone()
method is called by the file-system tree walk code for each file and
directory. We keep all state needed while indexing, and finally call
the methods to purge the db of stale entries and create the stemming
databases.
Single file(s) indexing: no database purging or stem db updating.
*/
class DbIndexer : public FsTreeWalkerCB {
public:
/** Constructor does nothing but store parameters */
DbIndexer(RclConfig *cnf, const std::string &dbd)
: config(cnf), dbdir(dbd) {
}
virtual ~DbIndexer();
/** Top level file system tree index method for updating a
given database.
The list is supposed to have all the filename space for the
db, and we shall purge entries for non-existing files at the
end. We create the temporary directory, open the database,
then call a file system walk for each top-level directory.
When walking is done, we create the stem databases and close
the main db.
*/
bool indexDb(bool resetbefore, std::list<std::string> *topdirs);
/** Index a list of files. No db cleaning or stemdb updating */
bool indexFiles(const std::list<std::string> &files);
/** Tree walker callback method */
FsTreeWalker::Status
processone(const std::string &, const struct stat *,
FsTreeWalker::CbFlag);
private:
FsTreeWalker walker;
RclConfig *config;
std::string dbdir;
Rcl::Db db;
std::string tmpdir;
bool init(bool rst = false);
};
#endif /* _INDEXER_H_INCLUDED_ */
src/index/recollindex.cpp
+70 -12
View File
@@ -1,20 +1,53 @@
#ifndef lint
static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.12 2005-11-30 09:46:25 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: recollindex.cpp,v 1.13 2005-12-14 11:00:48 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
#include <stdio.h>
#include <signal.h>
#include <sys/stat.h>
#include <iostream>
#include <list>
#include <string>
using namespace std;
#include "debuglog.h"
#include "rclinit.h"
#include "indexer.h"
#include "smallut.h"
#include "pathut.h"
ConfIndexer *indexer;
ConfIndexer *confindexer;
DbIndexer *dbindexer;
bool indexfiles(RclConfig *config, const list<string> &filenames)
{
if (filenames.empty())
return true;
// Note that we do not bother to check for multiple databases,
// which are currently a fiction anyway.
config->setKeyDir(path_getfather(*filenames.begin()));
string dbdir;
if (!config->getConfParam("dbdir", dbdir)) {
LOGERR(("indexfiles: no database directory in "
"configuration for %s\n", filenames.begin()->c_str()));
return false;
}
dbdir = path_tildexpand(dbdir);
dbindexer = new DbIndexer(config, dbdir);
return dbindexer->indexFiles(filenames);
}
static void cleanup()
{
delete indexer;
indexer = 0;
delete confindexer;
confindexer = 0;
delete dbindexer;
dbindexer = 0;
}
static void sigcleanup(int sig)
@@ -29,12 +62,16 @@ static int op_flags;
#define OPT_MOINS 0x1
#define OPT_z 0x2
#define OPT_h 0x4
#define OPT_i 0x8
static const char usage [] =
" recollindex [-hz]\n"
" recollindex [-hz] \n"
" recollindex -i <filename [filename ...]>\n"
"Options:\n"
" -h : print this message\n"
" -z : reset database before starting indexation\n\n"
" -i <filename [filename ...]> : index individual files. No db purge or stem\n"
" database updates in this case\n"
;
static void
@@ -59,22 +96,43 @@ int main(int argc, const char **argv)
switch (*(*argv)++) {
case 'z': op_flags |= OPT_z; break;
case 'h': op_flags |= OPT_h; break;
case 'i': op_flags |= OPT_i; break;
default: Usage(); break;
}
b1: argc--; argv++;
}
if (op_flags & OPT_h)
Usage();
if ((op_flags & OPT_z) && (op_flags & OPT_i))
Usage();
string reason;
RclConfig *config = recollinit(cleanup, sigcleanup, reason);
if (config == 0 || !config->ok()) {
string str = "Configuration problem: ";
str += reason;
fprintf(stderr, "%s\n", str.c_str());
cerr << "Configuration problem: " << reason << endl;
exit(1);
}
indexer = new ConfIndexer(config);
exit(!indexer->index((op_flags & OPT_z) != 0));
if (op_flags & OPT_i) {
list<string> filenames;
if (argc == 0) {
// Read from stdin
char line[1024];
while (fgets(line, 1023, stdin)) {
string sl(line);
trimstring(sl, "\n\r");
filenames.push_back(sl);
}
} else {
while (argc--) {
filenames.push_back(*argv++);
}
}
exit(!indexfiles(config, filenames));
} else {
confindexer = new ConfIndexer(config);
bool rezero(op_flags & OPT_z);
exit(!confindexer->index(rezero));
}
}