doc and comments
This commit is contained in:
+5
-4
@@ -1866,10 +1866,11 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi, Doc &doc)
|
||||
RECORD_APPEND(record, string(cstr_mbreaks), multibreaks.str());
|
||||
}
|
||||
|
||||
// If the file's md5 was computed, add value and term.
|
||||
// The value is optionally used for query result duplicate elimination,
|
||||
// and the term to find the duplicates.
|
||||
// We don't do this for empty docs.
|
||||
// If the file's md5 was computed, add value and term. The
|
||||
// value is optionally used for query result duplicate
|
||||
// elimination, and the term to find the duplicates (XM is the
|
||||
// prefix for rclmd5 in fields) We don't do this for empty
|
||||
// docs.
|
||||
const string *md5;
|
||||
if (doc.peekmeta(Doc::keymd5, &md5) && !md5->empty() &&
|
||||
md5->compare(cstr_md5empty)) {
|
||||
|
||||
+23
-41
@@ -1,4 +1,4 @@
|
||||
/* Copyright (C) 2013 J.F.Dockes
|
||||
/* Copyright (C) 2013-2020 J.F.Dockes
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
@@ -35,36 +35,36 @@ using namespace std;
|
||||
namespace Rcl {
|
||||
|
||||
/** Retrieve the dups of a given document. The input has to be a query result
|
||||
* because we use the xdocid. We get the md5 from this, then the dups */
|
||||
* because we use the xdocid. We get the md5 from this, then the dups */
|
||||
bool Db::docDups(const Doc& idoc, vector<Doc>& odocs)
|
||||
{
|
||||
if (m_ndb == 0) {
|
||||
LOGERR("Db::docDups: no db\n" );
|
||||
return false;
|
||||
LOGERR("Db::docDups: no db\n");
|
||||
return false;
|
||||
}
|
||||
if (idoc.xdocid == 0) {
|
||||
LOGERR("Db::docDups: null xdocid in input doc\n" );
|
||||
return false;
|
||||
LOGERR("Db::docDups: null xdocid in input doc\n");
|
||||
return false;
|
||||
}
|
||||
// Get the xapian doc
|
||||
Xapian::Document xdoc;
|
||||
XAPTRY(xdoc = m_ndb->xrdb.get_document(Xapian::docid(idoc.xdocid)),
|
||||
m_ndb->xrdb, m_reason);
|
||||
m_ndb->xrdb, m_reason);
|
||||
if (!m_reason.empty()) {
|
||||
LOGERR("Db::docDups: xapian error: " << (m_reason) << "\n" );
|
||||
return false;
|
||||
LOGERR("Db::docDups: xapian error: " << m_reason << "\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get the md5
|
||||
string digest;
|
||||
XAPTRY(digest = xdoc.get_value(VALUE_MD5), m_ndb->xrdb, m_reason);
|
||||
if (!m_reason.empty()) {
|
||||
LOGERR("Db::docDups: xapian error: " << (m_reason) << "\n" );
|
||||
return false;
|
||||
LOGERR("Db::docDups: xapian error: " << m_reason << "\n");
|
||||
return false;
|
||||
}
|
||||
if (digest.empty()) {
|
||||
LOGDEB("Db::docDups: doc has no md5\n" );
|
||||
return false;
|
||||
LOGDEB("Db::docDups: doc has no md5\n");
|
||||
return false;
|
||||
}
|
||||
string md5;
|
||||
MD5HexPrint(digest, md5);
|
||||
@@ -72,45 +72,27 @@ bool Db::docDups(const Doc& idoc, vector<Doc>& odocs)
|
||||
SearchData *sdp = new SearchData();
|
||||
std::shared_ptr<SearchData> sd(sdp);
|
||||
SearchDataClauseSimple *sdc =
|
||||
new SearchDataClauseSimple(SCLT_AND, md5, "rclmd5");
|
||||
new SearchDataClauseSimple(SCLT_AND, md5, "rclmd5");
|
||||
sdc->addModifier(SearchDataClause::SDCM_CASESENS);
|
||||
sdc->addModifier(SearchDataClause::SDCM_DIACSENS);
|
||||
sd->addClause(sdc);
|
||||
Query query(this);
|
||||
query.setCollapseDuplicates(0);
|
||||
if (!query.setQuery(sd)) {
|
||||
LOGERR("Db::docDups: setQuery failed\n" );
|
||||
return false;
|
||||
LOGERR("Db::docDups: setQuery failed\n");
|
||||
return false;
|
||||
}
|
||||
int cnt = query.getResCnt();
|
||||
for (int i = 0; i < cnt; i++) {
|
||||
Doc doc;
|
||||
if (!query.getDoc(i, doc)) {
|
||||
LOGERR("Db::docDups: getDoc failed at " << (i) << " (cnt " << (cnt) << ")\n" );
|
||||
return false;
|
||||
}
|
||||
odocs.push_back(doc);
|
||||
Doc doc;
|
||||
if (!query.getDoc(i, doc)) {
|
||||
LOGERR("Db::docDups: getDoc failed at " << i << " (cnt " << cnt <<
|
||||
")\n");
|
||||
return false;
|
||||
}
|
||||
odocs.push_back(doc);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#if 0
|
||||
{
|
||||
vector<Doc> dups;
|
||||
bool ret;
|
||||
LOGDEB("DOCDUPS\n" );
|
||||
ret = m_db->docDups(doc, dups);
|
||||
if (!ret) {
|
||||
LOGDEB("docDups failed\n" );
|
||||
} else if (dups.size() == 1) {
|
||||
LOGDEB("No dups\n" );
|
||||
} else {
|
||||
for (unsigned int i = 0; i < dups.size(); i++) {
|
||||
LOGDEB("Dup: " << (dups[i].url) << "\n" );
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user