internals: virtualized the doc fetcher interface
This commit is contained in:
Jean-Francois Dockes
@@ -0,0 +1,70 @@
|
||||
/* Copyright (C) 2012 J.F.Dockes
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "autoconfig.h"
|
||||
#endif
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include "debuglog.h"
|
||||
#include "rcldoc.h"
|
||||
|
||||
#include "fetcher.h"
|
||||
#include "bglfetcher.h"
|
||||
#include "debuglog.h"
|
||||
#include "ptmutex.h"
|
||||
#include "beaglequeuecache.h"
|
||||
|
||||
// We use a single beagle cache object to access beagle data. We protect it
|
||||
// against multiple thread access.
|
||||
static PTMutexInit o_beagler_mutex;
|
||||
|
||||
bool BGLDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
|
||||
{
|
||||
string udi;
|
||||
if (!idoc.getmeta(Rcl::Doc::keyudi, &udi) || udi.empty()) {
|
||||
LOGERR(("BGLDocFetcher:: no udi in idoc\n"));
|
||||
return false;
|
||||
}
|
||||
string data;
|
||||
Rcl::Doc dotdoc;
|
||||
{
|
||||
PTMutexLocker locker(o_beagler_mutex);
|
||||
// Retrieve from our webcache (beagle data). The beagler
|
||||
// object is created at the first call of this routine and
|
||||
// deleted when the program exits.
|
||||
static BeagleQueueCache o_beagler(cnf);
|
||||
if (!o_beagler.getFromCache(udi, dotdoc, data)) {
|
||||
LOGINFO(("BGLDocFetcher::fetch: failed for [%s]\n", udi.c_str()));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (dotdoc.mimetype.compare(idoc.mimetype)) {
|
||||
LOGINFO(("BGLDocFetcher:: udi [%s], mimetp mismatch: in: [%s], bgl "
|
||||
"[%s]\n", idoc.mimetype.c_str(), dotdoc.mimetype.c_str()));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool BGLDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig)
|
||||
{
|
||||
// Bgl sigs are empty
|
||||
sig.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
/* Copyright (C) 2012 J.F.Dockes
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifndef _BGLFETCHER_H_INCLUDED_
|
||||
#define _BGLFETCHER_H_INCLUDED_
|
||||
#include "fetcher.h"
|
||||
|
||||
/**
|
||||
* The Beagle cache fetcher:
|
||||
*/
|
||||
class BGLDocFetcher : public DocFetcher{
|
||||
virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out);
|
||||
virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig);
|
||||
virtual ~BGLDocFetcher() {}
|
||||
};
|
||||
|
||||
#endif /* _BGLFETCHER_H_INCLUDED_ */
|
||||
@@ -0,0 +1,43 @@
|
||||
/* Copyright (C) 2012 J.F.Dockes
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "autoconfig.h"
|
||||
#endif
|
||||
|
||||
#include "debuglog.h"
|
||||
|
||||
#include "fetcher.h"
|
||||
#include "fsfetcher.h"
|
||||
#include "bglfetcher.h"
|
||||
|
||||
DocFetcher *docFetcherMake(const Rcl::Doc& idoc)
|
||||
{
|
||||
if (idoc.url.empty()) {
|
||||
LOGERR(("docFetcherMakeg:: no url in doc!\n"));
|
||||
return false;
|
||||
}
|
||||
string backend;
|
||||
idoc.getmeta(Rcl::Doc::keybcknd, &backend);
|
||||
if (backend.empty() || !backend.compare("FS")) {
|
||||
return new FSDocFetcher;
|
||||
} else if (!backend.compare("BGL")) {
|
||||
return new BGLDocFetcher;
|
||||
} else {
|
||||
LOGERR(("DocFetcherFactory: unknown backend [%s]\n", backend.c_str()));
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
/* Copyright (C) 2012 J.F.Dockes
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifndef _FETCHER_H_INCLUDED_
|
||||
#define _FETCHER_H_INCLUDED_
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <string>
|
||||
|
||||
#include "rcldoc.h"
|
||||
|
||||
class RclConfig;
|
||||
|
||||
/**
|
||||
* Definition for a generic method to retrieve the data
|
||||
* for a document designated by its index data (udi/ipath/url).
|
||||
* This is used to retrieve the data for previewing. The
|
||||
* actual implementation is specific to the kind of backend (file
|
||||
* system, beagle cache, others in the future ?), and of course may
|
||||
* share code with the indexing-time functions from the specific backend.
|
||||
*/
|
||||
class DocFetcher {
|
||||
public:
|
||||
/** A RawDoc is the data for a source document either as a memory
|
||||
block, or pointed to by a file name */
|
||||
struct RawDoc {
|
||||
enum RawDocKind {RDK_FILENAME, RDK_DATA};
|
||||
RawDocKind kind;
|
||||
std::string data; // Doc data or file name
|
||||
struct stat st; // Only used if RDK_FILENAME
|
||||
};
|
||||
|
||||
/**
|
||||
* Return the data for the requested document, either as a
|
||||
* file-system file or as a memory object (maybe stream too in the
|
||||
* future?)
|
||||
* @param cnf the global config
|
||||
* @param idoc the data gathered from the index for this doc (udi/ipath)
|
||||
* @param out we may return either a file name or the document data.
|
||||
*/
|
||||
virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out) = 0;
|
||||
|
||||
/**
|
||||
* Return the signature for the requested document. This is used for
|
||||
* up-to-date tests performed out of indexing (e.g.: verifying that a
|
||||
* document is not stale before previewing it).
|
||||
* @param cnf the global config
|
||||
* @param idoc the data gathered from the index for this doc (udi/ipath)
|
||||
* @param sig output.
|
||||
*/
|
||||
virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig) = 0;
|
||||
virtual ~DocFetcher() {}
|
||||
};
|
||||
|
||||
/** Returns an appropriate fetcher object given the backend string identifier */
|
||||
DocFetcher *docFetcherMake(const Rcl::Doc& idoc);
|
||||
|
||||
#endif /* _FETCHER_H_INCLUDED_ */
|
||||
@@ -0,0 +1,70 @@
|
||||
/* Copyright (C) 2012 J.F.Dockes
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "autoconfig.h"
|
||||
#endif
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include "debuglog.h"
|
||||
#include "cstr.h"
|
||||
|
||||
#include "fetcher.h"
|
||||
#include "fsfetcher.h"
|
||||
#include "fsindexer.h"
|
||||
#include "debuglog.h"
|
||||
|
||||
using std::string;
|
||||
|
||||
static bool urltopath(const Rcl::Doc& idoc, string& fn, struct stat& st)
|
||||
{
|
||||
// The url has to be like file://
|
||||
if (idoc.url.find(cstr_fileu) != 0) {
|
||||
LOGERR(("FSDocFetcher::fetch/sig: non fs url: [%s]\n",
|
||||
idoc.url.c_str()));
|
||||
return false;
|
||||
}
|
||||
fn = idoc.url.substr(7, string::npos);
|
||||
if (stat(fn.c_str(), &st) < 0) {
|
||||
LOGERR(("FSDocFetcher::fetch: stat errno %d for [%s]\n",
|
||||
errno, fn.c_str()));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool FSDocFetcher::fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out)
|
||||
{
|
||||
string fn;
|
||||
if (!urltopath(idoc, fn, out.st))
|
||||
return false;
|
||||
out.kind = RawDoc::RDK_FILENAME;
|
||||
out.data = fn;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool FSDocFetcher::makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig)
|
||||
{
|
||||
string fn;
|
||||
struct stat st;
|
||||
if (!urltopath(idoc, fn, st))
|
||||
return false;
|
||||
FsIndexer::makesig(&st, sig);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
/* Copyright (C) 2012 J.F.Dockes
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the
|
||||
* Free Software Foundation, Inc.,
|
||||
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
*/
|
||||
#ifndef _FSFETCHER_H_INCLUDED_
|
||||
#define _FSFETCHER_H_INCLUDED_
|
||||
|
||||
#include "fetcher.h"
|
||||
|
||||
/**
|
||||
* The file-system fetcher:
|
||||
*/
|
||||
class FSDocFetcher : public DocFetcher{
|
||||
/** FSDocFetcher::fetch always returns a file name */
|
||||
virtual bool fetch(RclConfig* cnf, const Rcl::Doc& idoc, RawDoc& out);
|
||||
|
||||
/** Calls stat to retrieve file signature data */
|
||||
virtual bool makesig(RclConfig* cnf, const Rcl::Doc& idoc, string& sig);
|
||||
virtual ~FSDocFetcher() {}
|
||||
};
|
||||
|
||||
#endif /* _FSFETCHER_H_INCLUDED_ */
|
||||
Reference in New Issue
Block a user