*** empty log message ***
This commit is contained in:
dockes
Executable
+114
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python
|
||||
__doc__ = """
|
||||
''Lookup'' notes file indexing
|
||||
|
||||
The file format has text notes separated by lines with a single '%' character
|
||||
|
||||
If the script is called with just the file name as an argument, it will
|
||||
(re)index the contents.
|
||||
|
||||
If the script is called with second numeric argument, it will retrieve the
|
||||
specified record and output it in html
|
||||
"""
|
||||
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import re
|
||||
|
||||
rclconf = "/Users/dockes/.recoll-test"
|
||||
|
||||
def udi(docfile, numrec):
|
||||
return docfile + "#" + str(numrec)
|
||||
|
||||
###############################################################
|
||||
def index_rec(db, numrec, rec):
|
||||
doc = recollq.Doc()
|
||||
# url
|
||||
doc.url = "file://" + docfile
|
||||
# utf8fn
|
||||
# ipath
|
||||
doc.ipath = str(numrec)
|
||||
# mimetype
|
||||
doc.mimetype = "text/plain"
|
||||
# mtime
|
||||
# origcharset
|
||||
# title
|
||||
lines = rec.split("\n")
|
||||
if len(lines) >= 2:
|
||||
doc.title = unicode(lines[1], "iso-8859-1")
|
||||
if len(doc.title.strip()) == 0 and len(lines) >= 3:
|
||||
doc.title = unicode(lines[2], "iso-8859-1")
|
||||
# keywords
|
||||
# abstract
|
||||
# author
|
||||
# fbytes
|
||||
doc.fbytes = str(fbytes)
|
||||
# text
|
||||
doc.text = unicode(rec, "iso-8859-1")
|
||||
# dbytes
|
||||
doc.dbytes = str(len(rec))
|
||||
# sig
|
||||
if numrec == 0:
|
||||
doc.sig = str(fmtime)
|
||||
db.addOrUpdate(udi(docfile, numrec), u"", doc)
|
||||
|
||||
def output_rec(rec):
|
||||
# Escape html
|
||||
rec = unicode(rec, "iso-8859-1").encode("utf-8")
|
||||
rec = rec.replace("<", "<");
|
||||
rec = rec.replace("&", "&");
|
||||
rec = rec.replace('"', "&dquot;");
|
||||
print '<html><head>'
|
||||
print '<meta http-equiv="Content-Type" content="text/html;charset=UTF-8">'
|
||||
print '</head><body><pre>'
|
||||
print rec
|
||||
print '</pre></body></html>'
|
||||
|
||||
|
||||
################################################################
|
||||
|
||||
def usage():
|
||||
sys.stderr.write("Usage: doclookup.py <filename> [<recnum>]\n")
|
||||
exit(1)
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
usage()
|
||||
|
||||
docfile = sys.argv[1]
|
||||
|
||||
if len(sys.argv) > 2:
|
||||
targetnum = int(sys.argv[2])
|
||||
else:
|
||||
targetnum = None
|
||||
|
||||
#print docfile, targetnum
|
||||
|
||||
stdata = os.stat(docfile)
|
||||
fmtime = stdata[stat.ST_MTIME]
|
||||
fbytes = stdata[stat.ST_SIZE]
|
||||
f = open(docfile, 'r')
|
||||
|
||||
if targetnum == None:
|
||||
import recollq
|
||||
db = recollq.connect(confdir=rclconf, writable=1)
|
||||
if not db.needUpdate(udi(docfile, 0), str(fmtime)):
|
||||
exit(0)
|
||||
|
||||
rec = ""
|
||||
numrec = 1
|
||||
for line in f:
|
||||
if re.compile("^%[ \t]*").match(line):
|
||||
if targetnum == None:
|
||||
index_rec(db, numrec, rec)
|
||||
elif targetnum == numrec:
|
||||
output_rec(rec)
|
||||
exit(0)
|
||||
numrec += 1
|
||||
rec = ""
|
||||
else:
|
||||
rec += line
|
||||
|
||||
if targetnum == None:
|
||||
index_rec(db, 0, "")
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import mailbox
|
||||
import email.header
|
||||
import email.utils
|
||||
#import sys
|
||||
import recollq
|
||||
import os
|
||||
import stat
|
||||
|
||||
#mbfile = "/Users/dockes/projets/fulltext/testrecoll/mail/fred"
|
||||
mbfile = "/Users/dockes/mail/outmail"
|
||||
rclconf = "/Users/dockes/.recoll-test"
|
||||
|
||||
def header_value(msg, nm, to_utf = False):
|
||||
value = msg.get(nm)
|
||||
if value == None:
|
||||
return ""
|
||||
value = value.replace("\n", "")
|
||||
value = value.replace("\r", "")
|
||||
#print value
|
||||
parts = email.header.decode_header(value)
|
||||
#print parts
|
||||
univalue = u""
|
||||
for part in parts:
|
||||
if part[1] != None:
|
||||
univalue += unicode(part[0], part[1]) + " "
|
||||
else:
|
||||
univalue += part[0] + " "
|
||||
if to_utf:
|
||||
return univalue.encode('utf-8')
|
||||
else:
|
||||
return univalue
|
||||
|
||||
class mbox_indexer:
|
||||
def __init__(self, mbfile):
|
||||
self.mbfile = mbfile
|
||||
stdata = os.stat(mbfile)
|
||||
self.fmtime = stdata[stat.ST_MTIME]
|
||||
self.fbytes = stdata[stat.ST_SIZE]
|
||||
self.msgnum = 1
|
||||
|
||||
def sig(self):
|
||||
return str(self.fmtime) + ":" + str(self.fbytes)
|
||||
def udi(self, msgnum):
|
||||
return self.mbfile + ":" + str(msgnum)
|
||||
|
||||
def index(self, db):
|
||||
if not db.needUpdate(self.udi(1), self.sig()):
|
||||
return None
|
||||
mb = mailbox.mbox(self.mbfile)
|
||||
for msg in mb.values():
|
||||
self.index_message(db, msg)
|
||||
self.msgnum += 1
|
||||
|
||||
def index_message(self, db, msg):
|
||||
doc = recollq.Doc()
|
||||
doc.author = header_value(msg, "From")
|
||||
# url
|
||||
doc.url = "file://" + self.mbfile
|
||||
# utf8fn
|
||||
# ipath
|
||||
doc.ipath = str(self.msgnum)
|
||||
# mimetype
|
||||
doc.mimetype = "message/rfc822"
|
||||
# mtime
|
||||
dte = header_value(msg, "Date")
|
||||
tm = email.utils.parsedate_tz(dte)
|
||||
if tm == None:
|
||||
doc.mtime = str(self.fmtime)
|
||||
else:
|
||||
doc.mtime = str(email.utils.mktime_tz(tm))
|
||||
# origcharset
|
||||
# title
|
||||
doc.title = header_value(msg, "Subject")
|
||||
# keywords
|
||||
# abstract
|
||||
# author
|
||||
# fbytes
|
||||
doc.fbytes = str(self.fbytes)
|
||||
# text
|
||||
text = u""
|
||||
text += u"From: " + header_value(msg, "From") + u"\n"
|
||||
text += u"To: " + header_value(msg, "To") + u"\n"
|
||||
text += u"Subject: " + header_value(msg, "Subject") + u"\n"
|
||||
#text += u"Message-ID: " + header_value(msg, "Message-ID") + u"\n"
|
||||
text += u"\n"
|
||||
for part in msg.walk():
|
||||
if part.is_multipart():
|
||||
pass #print "Multipart: " + part.get_content_type()
|
||||
else:
|
||||
ct = part.get_content_type()
|
||||
#print "Simple: " + ct
|
||||
if ct.lower() == "text/plain":
|
||||
charset = part.get_content_charset("iso-8859-1")
|
||||
text += unicode(part.get_payload(None, True), charset)
|
||||
doc.text = text
|
||||
# dbytes
|
||||
doc.dbytes = str(len(text))
|
||||
# sig
|
||||
doc.sig = self.sig()
|
||||
udi = self.udi(self.msgnum)
|
||||
db.addOrUpdate(udi, u"", doc)
|
||||
|
||||
|
||||
db = recollq.connect(confdir=rclconf, writable=1)
|
||||
|
||||
mbidx = mbox_indexer(mbfile)
|
||||
mbidx.index(db)
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import recollq
|
||||
allmeta = ("title", "keywords", "abstract", "url", "mimetype", "mtime",
|
||||
"ipath", "fbytes", "dbytes", "relevance")
|
||||
|
||||
|
||||
def dotest(db, q):
|
||||
query = db.query()
|
||||
#query1 = db.query()
|
||||
|
||||
nres = query.execute(q)
|
||||
print "Result count: ", nres
|
||||
if nres > 10:
|
||||
nres = 10
|
||||
while query.next >= 0 and query.next < nres:
|
||||
doc = query.fetchone()
|
||||
print query.next
|
||||
for k in ("title",):
|
||||
print k, ":", getattr(doc, k).encode('utf-8')
|
||||
abs = db.makeDocAbstract(doc, query).encode('utf-8')
|
||||
print abs
|
||||
print
|
||||
|
||||
# End dotest
|
||||
|
||||
q = "essaouira"
|
||||
|
||||
print "TESTING WITH .recoll"
|
||||
db = recollq.connect()
|
||||
db.setAbstractParams(maxchars=80, contextwords=2)
|
||||
dotest(db, q)
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
print "TESTING WITH .recoll-test"
|
||||
db = recollq.connect(confdir="/Users/dockes/.recoll-test")
|
||||
dotest(db, q)
|
||||
|
||||
print "TESTING WITH .recoll-doc"
|
||||
db = recollq.connect(confdir="/y/home/dockes/.recoll-doc")
|
||||
dotest(db, q)
|
||||
|
||||
print "TESTING WITH .recoll and .recoll-doc"
|
||||
db = recollq.connect(confdir="/Users/dockes/.recoll",
|
||||
extra_dbs=("/y/home/dockes/.recoll-doc",))
|
||||
dotest(db, q)
|
||||
|
||||
Reference in New Issue
Block a user