ckpt: pst: basic indexing of email. no getipath/preview

src/filters/rclpst.py

@@ -20,7 +20,8 @@
 #
 # Process the stream produced by a modified pffexport:
 # https://github.com/libyal/libpff
-# The tool has been modified to produce a data stream instead of a file tree
+# The modification allows producing a data stream instead of a file tree
+#
 
 import sys
 import os
@@ -31,20 +32,21 @@ import traceback
 import email.parser
 import email.policy
 import mailbox
+import subprocess
 
+import rclexecm
 import rclconfig
 import conftree
 
-def _deb(s):
-	print("%s"%s, file=sys.stderr)
-
 
 # The pffexport stream yields the email in several pieces, with some
 # data missing (e.g. attachment MIME types). We rebuild a complete
 # message for parsing by the Recoll email handler
 class EmailBuilder(object):
-	def __init__(self):
+    def __init__(self, logger, mimemap):
+        self.log = logger
         self.reset()
+        self.mimemap = mimemap
         self.parser = email.parser.Parser(policy = email.policy.default)
     def reset(self):
         self.headers = ''
@@ -58,19 +60,18 @@ class EmailBuilder(object):
         self.bodymimemain = main
         self.bodymimesub = sub
     def addattachment(self, att, filename):
-		_deb("Adding attachment")
+        self.log("Adding attachment")
         self.attachments.append((att, filename))
     def flush(self):
-		if not self.headers:
+        if not (self.headers and (self.body or self.attachments)):
-			_deb("Not flushing because no headers")
+            self.log("Not flushing because no headers or no body/attach")
-		if self.headers and (self.body or self.attachments):
+            return None
-			newmsg = email.message.EmailMessage(policy =
+        newmsg = email.message.EmailMessage(policy=email.policy.default)
-													email.policy.default)
             
         headerstr = self.headers.decode('utf-8')
         # print("%s" % headerstr)
         headers = self.parser.parsestr(headerstr, headersonly=True)
-			_deb("EmailBuilder: content-type %s" % headers['content-type'])
+        #self.log("EmailBuilder: content-type %s" % headers['content-type'])
         for nm in ('from', 'subject'):
             if nm in headers:
                 newmsg.add_header(nm, headers[nm])
@@ -80,7 +81,7 @@ class EmailBuilder(object):
         for toheader in tolist:
             for dest in toheader.addresses:
                 sd = str(dest).replace('\n', '').replace('\r','')
-					_deb("EmailBuilder: dest %s" % sd)
+                #self.log("EmailBuilder: dest %s" % sd)
                 alldests += sd + ", "
             alldests = alldests.rstrip(", ")
             newmsg.add_header('to', alldests)
@@ -92,53 +93,40 @@ class EmailBuilder(object):
                                subtype = self.bodymimesub)
                 
         for att in self.attachments:
-				#if self.body:
+            fn = att[1]
-				#	newmsg.make_mixed()
+            ext = os.path.splitext(fn)[1]
-				ext = os.path.splitext(att[1])[1]
+            mime = self.mimemap.get(ext)
-				_deb("Querying mimemap with %s" % ext)
-				mime = mimemap.get(ext)
             if not mime:
                 mime = 'application/octet-stream'
-				_deb("Attachment: filename %s MIME %s" % (att[1], mime))
+            #self.log("Attachment: filename %s MIME %s" % (fn, mime))
             mt,st = mime.split('/')
             newmsg.add_attachment(att[0], maintype=mt, subtype=st,
-									  filename=att[1])
+                                  filename=fn)
 
-			newmsg.set_unixfrom("From some@place.org Sun Jan 01 00:00:00 2000")
+        #newmsg.set_unixfrom("From some@place.org Sun Jan 01 00:00:00 2000")
-			print("%s\n" % newmsg.as_string(unixfrom=True, maxheaderlen=80))
+        #print("%s\n" % newmsg.as_string(unixfrom=True, maxheaderlen=80))
+        ret = newmsg.as_string(maxheaderlen=100)
 
         self.reset()
+        return ret
     
 
 class PFFReader(object):
-	def __init__(self, infile=sys.stdin):
+    def __init__(self, logger, infile=sys.stdin):
-		try:
+        self.log = logger
-			self.myname = os.path.basename(sys.argv[0])
+        config = rclconfig.RclConfig()
-		except:
+        dir1 = os.path.join(config.getConfDir(), "examples")
-			self.myname = "???"
+        dir2 = os.path.join(config.datadir, "examples")
-
+        self.mimemap = conftree.ConfStack('mimemap', [dir1, dir2])
         self.infile = infile
         self.fields = {}
-		self.msg = EmailBuilder()
+        self.msg = EmailBuilder(self.log, self.mimemap)
-		
-		if sys.platform == "win32":
-			import msvcrt
-			msvcrt.setmode(self.outfile.fileno(), os.O_BINARY)
-			msvcrt.setmode(self.infile.fileno(), os.O_BINARY)
-		self.debugfile = None
-		if self.debugfile:
-			self.errfout = open(self.debugfile, "a")
-		else:
-			self.errfout = sys.stderr
-		
-	def log(self, s):
-		print("PFFReader: %s: %s" % (self.myname, s), file=self.errfout)
         
     # Read single parameter from process input: line with param name and size
     # followed by data. The param name is returned as str/unicode, the data
     # as bytes
     def readparam(self):
-		inf = self.infile.buffer
+        inf = self.infile
         s = inf.readline()
         if s == b'':
             return ('', b'')
@@ -163,6 +151,7 @@ class PFFReader(object):
 
     def mainloop(self):
         basename = ''
+        path = ''
         while 1:
             name, data = self.readparam()
             if name == "":
@@ -173,9 +162,9 @@ class PFFReader(object):
                 paramstr = ''
 
             if name == 'filename':
-				basename = os.path.basename(paramstr)
+                self.log("filename: %s" %  paramstr)
-				self.log("name: [%s] data: %s" %
+                path = paramstr
-						 (name, paramstr))
+                basename = os.path.basename(path)
                 parentdir = os.path.basename(os.path.dirname(paramstr))
             elif name == 'data':
                 if parentdir == 'Attachments':
@@ -183,9 +172,10 @@ class PFFReader(object):
                     self.msg.addattachment(data, basename)
                 else:
                     if basename == 'OutlookHeaders.txt':
-						self.msg.flush()
+                        doc = self.msg.flush()
-						pass
+                        if doc:
-					if basename == 'ConversationIndex.txt':
+                            yield((doc, path))
+                    elif basename == 'ConversationIndex.txt':
                         pass
                     elif basename == 'Recipients.txt':
                         pass
@@ -207,13 +197,73 @@ class PFFReader(object):
                 basename = ''
                 parentdir = ''
         self.log("Out of loop")
-		self.msg.flush()
+        doc = self.msg.flush()
+        if doc:
+            yield((doc, path))
+        return
 
 
-config = rclconfig.RclConfig()
+class PstExtractor(object):
-dir1 = os.path.join(config.getConfDir(), "examples")
+    def __init__(self, em):
-dir2 = os.path.join(config.datadir, "examples")
+        self.currentindex = 0
-mimemap = conftree.ConfStack('mimemap', [dir1, dir2])
+        self.em = em
+        self.cmd = ["pffexport", "-q", "-t", "/nonexistent", "-s"]
 
-proto = PFFReader()
+    def startCmd(self, filename):
-proto.mainloop()
+        fullcmd = self.cmd + [rclexecm.subprocfile(filename)]
+        try:
+            self.proc = subprocess.Popen(fullcmd, stdout=subprocess.PIPE)
+        except subprocess.CalledProcessError as err:
+            self.em.rclog("Pst: Popen(%s) error: %s" % (fullcmd, err))
+            return False
+        except OSError as err:
+            self.em.rclog("Pst: Popen(%s) OS error: %s" % (fullcmd, err))
+            return (False, "")
+        self.filein = self.proc.stdout
+        return True
+
+    def extractone(self, ipath):
+        #self.em.rclog("extractone: [%s]" % ipath)
+        docdata = ""
+        ok = False
+        iseof = True
+        return (ok, docdata, rclexecm.makebytes(ipath), iseof)
+
+    ###### File type handler api, used by rclexecm ---------->
+    def openfile(self, params):
+        filename = params["filename:"]
+        
+        if not self.startCmd(filename):
+            return False
+        
+        reader = PFFReader(self.em.rclog, infile=self.filein)
+        self.generator = reader.mainloop()
+        return True
+
+
+    def getipath(self, params):
+        ipath = params["ipath:"]
+        ok, data, ipath, eof = self.extractone(ipath)
+        if ok:
+            return (ok, data, ipath, eof)
+        # Not found. Maybe we need to decode the path?
+        try:
+            ipath = ipath.decode("utf-8")
+            return self.extractone(ipath)
+        except Exception as err:
+            return (ok, data, ipath, eof)
+        
+    def getnext(self, params):
+        try:
+            doc, ipath = next(self.generator)
+            self.em.setmimetype("message/rfc822")
+            #self.em.rclog("doc %s ipath %s" % (doc[:40], ipath))
+        except StopIteration:
+            return(False, "", "", rclexecm.RclExecM.eofnow)
+        return (True, doc, ipath, False)
+    
+
+# Main program: create protocol handler and extractor and run them
+proto = rclexecm.RclExecM()
+extract = PstExtractor(proto)
+rclexecm.main(proto, extract)

src/python/recoll/recoll/rclconfig.py

@@ -78,6 +78,8 @@ class RclConfig:
 
     def getConfDir(self):
         return self.confdir
+    def getDataDir(self):
+        return self.datadir
     
     def setKeyDir(self, dir):
         self.keydir = dir

src/sampleconf/mimeconf

@@ -78,6 +78,7 @@ application/pdf = execm rclpdf.py
 application/postscript = exec rclps
 application/sql = internal text/plain
 application/vnd.ms-excel = execm rclxls.py
+application/vnd.ms-outlook = execm rclpst.py
 application/vnd.ms-powerpoint = execm rclppt.py
 application/vnd.oasis.opendocument.text = internal xsltproc meta.xml opendoc-meta.xsl content.xml opendoc-body.xsl
 application/vnd.oasis.opendocument.text-template = internal xsltproc meta.xml opendoc-meta.xsl content.xml opendoc-body.xsl

src/sampleconf/mimemap

@@ -61,6 +61,9 @@
 # extracted message. Also used by Windows Live Mail
 .eml = message/rfc822
 
+.pst = application/vnd.ms-outlook
+.ost = application/vnd.ms-outlook
+
 .pdf = application/pdf
 
 .ps = application/postscript