tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

autosuffs featurelet

Browse Source
This commit is contained in:
Jean-Francois Dockes
2010-09-10 09:50:11 +02:00
parent f3b0b49c77
commit 176b9b19b6
10 changed files with 201 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/smallut.cpp
+59 -20
View File
@@ -174,7 +174,8 @@ bool samecharset(const string &cs1, const string &cs2)
return mcs1 == mcs2;
}
template <class T> bool stringToStrings(const string &s, T &tokens)
template <class T> bool stringToStrings(const string &s, T &tokens,
const string& addseps)
{
string current;
tokens.clear();
@@ -237,17 +238,34 @@ template <class T> bool stringToStrings(const string &s, T &tokens)
break;
default:
switch(state) {
case ESCAPE:
state = INQUOTE;
break;
case SPACE:
state = TOKEN;
break;
case TOKEN:
case INQUOTE:
break;
}
if (!addseps.empty() && addseps.find(s[i]) != string::npos) {
switch(state) {
case ESCAPE:
state = INQUOTE;
break;
case INQUOTE:
break;
case SPACE:
tokens.insert(tokens.end(), string(1, s[i]));
continue;
case TOKEN:
tokens.insert(tokens.end(), current);
current.erase();
tokens.insert(tokens.end(), string(1, s[i]));
state = SPACE;
continue;
}
} else switch(state) {
case ESCAPE:
state = INQUOTE;
break;
case SPACE:
state = TOKEN;
break;
case TOKEN:
case INQUOTE:
break;
}
current += s[i];
}
}
@@ -263,17 +281,20 @@ template <class T> bool stringToStrings(const string &s, T &tokens)
}
return true;
}
bool stringToStrings(const string &s, list<string> &tokens)
bool stringToStrings(const string &s, list<string> &tokens,
const string& as)
{
return stringToStrings<list<string> >(s, tokens);
return stringToStrings<list<string> >(s, tokens, as);
}
bool stringToStrings(const string &s, vector<string> &tokens)
bool stringToStrings(const string &s, vector<string> &tokens,
const string& as)
{
return stringToStrings<vector<string> >(s, tokens);
return stringToStrings<vector<string> >(s, tokens, as);
}
bool stringToStrings(const string &s, set<string> &tokens)
bool stringToStrings(const string &s, set<string> &tokens,
const string& as)
{
return stringToStrings<set<string> >(s, tokens);
return stringToStrings<set<string> >(s, tokens, as);
}
template <class T> void stringsToString(const T &tokens, string &s)
@@ -729,10 +750,28 @@ struct spair suffpairs[] = {
};
int nsuffpairs = sizeof(suffpairs) / sizeof(struct spair);
const char *thisprog;
int main(int argc, char **argv)
{
#if 0
thisprog = *argv++;argc--;
#if 1
if (argc <=0 ) {
cerr << "Usage: smallut <stringtosplit>" << endl;
exit(1);
}
string s = *argv++;argc--;
vector<string> vs;
if (!stringToStrings(s, vs, ":-()")) {
cerr << "Bad entry" << endl;
exit(1);
}
for (vector<string>::const_iterator it = vs.begin(); it != vs.end(); it++)
cerr << "[" << *it << "] ";
cerr << endl;
exit(0);
#elif 0
for (int i = 0; i < npairs; i++) {
{
int c = stringicmp(pairs[i].s1, pairs[i].s2);
@@ -768,7 +807,7 @@ int main(int argc, char **argv)
cout << "[" << neutchars(testit, "\r\n") << "]" << endl;
string i, o;
cout << "neutchars(null) is [" << neutchars(i, "\r\n") << "]" << endl;
#elif 1
#elif 0
map<string, string> substs;
substs["a"] = "A_SUBST";
substs["title"] = "TITLE_SUBST";
src/utils/smallut.h
+25 -9
View File
@@ -35,6 +35,17 @@ using std::set;
// Note these are all ascii routines
extern int stringicmp(const string& s1, const string& s2);
// For find_if etc.
struct StringIcmpPred {
StringIcmpPred(const string& s1)
: m_s1(s1)
{}
bool operator()(const string& s2) {
return stringicmp(m_s1, s2) == 0;
}
const string& m_s1;
};
extern int stringlowercmp(const string& alreadylower, const string& s2);
extern int stringuppercmp(const string& alreadyupper, const string& s2);
extern void stringtolower(string& io);
@@ -52,11 +63,15 @@ extern bool samecharset(const string &cs1, const string &cs2);
* dquotes can be escaped with \ etc...
* Input is handled a byte at a time, things will work as long as space tab etc.
* have the ascii values and can't appear as part of a multibyte char. utf-8 ok
* but so are the iso-8859-x and surely others.
* but so are the iso-8859-x and surely others. addseps do have to be
* single-bytes
*/
extern bool stringToStrings(const string &s, list<string> &tokens);
extern bool stringToStrings(const string &s, vector<string> &tokens);
extern bool stringToStrings(const string &s, set<string> &tokens);
extern bool stringToStrings(const string& s, list<string> &tokens,
const string& addseps = "");
extern bool stringToStrings(const string& s, vector<string> &tokens,
const string& addseps = "");
extern bool stringToStrings(const string& s, set<string> &tokens,
const string& addseps = "");
/**
* Inverse operation:
@@ -78,7 +93,7 @@ extern bool stringToBool(const string &s);
tab}) at beginning and end of input string */
extern void trimstring(string &s, const char *ws = " \t");
/** Escape things like < or & by turining them to entities */
/** Escape things like < or & by turning them into entities */
extern string escapeHtml(const string &in);
/** Replace some chars with spaces (ie: newline chars). This is not utf8-aware
@@ -86,8 +101,8 @@ extern string escapeHtml(const string &in);
extern string neutchars(const string &str, const string &chars);
extern void neutchars(const string &str, string& out, const string &chars);
/** turn string into something that won't be expanded by a shell. In practise
* quote with single-quotes and escape internal singlequotes */
/** Turn string into something that won't be expanded by a shell. In practise
* quote with double-quotes and escape $`\ */
extern string escapeShell(const string &str);
/** Truncate a string to a given maxlength, avoiding cutting off midword
@@ -108,6 +123,7 @@ bool pcSubst(const string& in, string& out, map<char, string>& subs);
/** Substitute printf-like percents and also %(key) */
bool pcSubst(const string& in, string& out, map<string, string>& subs);
/** Compute times to help with perf issues */
class Chrono {
public:
Chrono();
@@ -130,8 +146,8 @@ class Chrono {
long m_nsecs;
};
class TempBuf {
public:
/** Temp buffer with automatic deallocation */
struct TempBuf {
TempBuf()
: m_buf(0)
{}