From a5a6b4112bee9ac92df36f1890b44783a237f811 Mon Sep 17 00:00:00 2001 From: Jean-Francois Dockes Date: Sat, 2 Jun 2018 18:14:35 +0200 Subject: [PATCH] Split rclconfig.py into rclconfig / conftree for sharing code with other users --- .gitignore | 2 + src/Makefile.am | 5 +- src/python/recoll/recoll/conftree.py | 253 ++++++++++++++++++++++++++ src/python/recoll/recoll/rclconfig.py | 126 +------------ 4 files changed, 261 insertions(+), 125 deletions(-) create mode 100644 src/python/recoll/recoll/conftree.py mode change 100755 => 100644 src/python/recoll/recoll/rclconfig.py diff --git a/.gitignore b/.gitignore index 0b77bd5f..920281f1 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ src/depcomp src/doc/user/usermanual.pdf src/doc/user/webhelp/docs/* src/doc/user/webhelp/xincluded-profiled.xml +src/filters/conftree.py +src/filters/rclconfig.py src/filters/rclexecm.pyc src/filters/rcllatinclass.pyc src/install-sh diff --git a/src/Makefile.am b/src/Makefile.am index 987c8a38..42f32f39 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -511,6 +511,7 @@ python/recoll/pyrclextract.cpp \ python/recoll/pyrecoll.cpp \ python/recoll/pyrecoll.h \ python/recoll/recoll/__init__.py \ +python/recoll/recoll/conftree.py \ python/recoll/recoll/rclconfig.py \ python/recoll/setup.py.in \ python/samples/docdups.py \ @@ -681,13 +682,15 @@ filters/ppt-dump.py \ filters/xls-dump.py \ filters/xlsxmltocsv.py \ filters/msodump.zip \ +python/recoll/recoll/conftree.py \ python/recoll/recoll/rclconfig.py install-data-hook: (cd $(DESTDIR)/$(filterdir); \ chmod a+x rcl* ppt-dump.py xls-dump.py xlsxmltocsv.py hotrecoll.py; \ chmod a+x recoll-we-move-files.py; \ - chmod 0644 msodump.zip rclexecm.py rcllatinstops.zip rclconfig.py rclmidi.py) + chmod 0644 msodump.zip rclexecm.py rcllatinstops.zip \ + rclconfig.py conftree.py rclmidi.py) if MAKEUSERDOC rdocdir = $(pkgdatadir)/doc diff --git a/src/python/recoll/recoll/conftree.py b/src/python/recoll/recoll/conftree.py new file mode 100644 index 00000000..4ddc6117 --- /dev/null +++ b/src/python/recoll/recoll/conftree.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python +# Copyright (C) 2016 J.F.Dockes +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the +# Free Software Foundation, Inc., +# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +from __future__ import print_function + +import locale +import re +import os +import sys +import base64 +import platform + +def _debug(s): + print("%s"%s, file=sys.stderr) + +class ConfSimple(object): + """A ConfSimple class reads a recoll configuration file, which is + a typical ini file (see the Recoll manual). It's a dictionary of + dictionaries which lets you retrieve named values from the top + level or a subsection""" + + def __init__(self, confname, tildexp = False, readonly = True): + self.submaps = {} + self.dotildexpand = tildexp + self.readonly = readonly + self.confname = confname + + try: + f = open(confname, 'rb') + except Exception as exc: + #_debug("Open Exception: %s" % exc) + # File does not exist -> empty config, not an error. + self.submaps = {} + self.submaps[b''] = {} + return + + self._parseinput(f) + + def _parseinput(self, f): + appending = False + line = b'' + submapkey = b'' + for cline in f: + cline = cline.rstrip(b'\r\n') + if appending: + line = line + cline + else: + line = cline + line = line.strip() + if line == b'' or line[0] == b'#'[0]: + continue + + if line[len(line)-1] == b'\\'[0]: + line = line[0:len(line)-1] + appending = True + continue + + appending = False + #_debug(line) + if line[0] == b'['[0]: + line = line.strip(b'[]') + if self.dotildexpand: + submapkey = os.path.expanduser(line) + if type(submapkey) == type(u''): + submapkey = submapkey.encode('utf-8') + else: + submapkey = line + #_debug("Submapkey: [%s]" % submapkey) + continue + + nm, sep, value = line.partition(b'=') + if sep == b'': + # No equal sign in line -> considered comment + continue + + nm = nm.strip() + value = value.strip() + #_debug("sk [%s] nm: [%s] value: [%s]" % (submapkey, nm, value)) + if not submapkey in self.submaps: + self.submaps[submapkey] = {} + self.submaps[submapkey][nm] = value + + def getbin(self, nm, sk = b''): + '''Returns None if not found, empty string if found empty''' + if type(nm) != type(b'') or type(sk) != type(b''): + raise TypeError("getbin: parameters must be binary not unicode") + #_debug("ConfSimple::getbin nm [%s] sk [%s]" % (nm, sk)) + if not sk in self.submaps: + return None + if not nm in self.submaps[sk]: + return None + return self.submaps[sk][nm] + + def get(self, nm, sk = b''): + dodecode = False + if type(nm) == type(u''): + dodecode = True + nm = nm.encode('utf-8') + if type(sk) == type(u''): + sk = sk.encode('utf-8') + #v = ConfSimple.getbin(self, nm, sk) + v = self.getbin(nm, sk) + if v and dodecode: + v = v.decode('utf-8') + return v + + def getNamesbin(self, sk = b''): + if not sk in self.submaps: + return None + return list(self.submaps[sk].keys()) + + def getNames(self, sk = ''): + if not sk in self.submaps: + return None + dodecode = False + if type(sk) == type(u''): + dodecode = True + sk = sk.encode('utf-8') + names = self.getNamesbin(sk) + if names and dodecode: + names = [nm.decode('utf-8') for nm in names] + return names + + def _rewrite(self): + if self.readonly: + raise Exception("ConfSimple is readonly") + + tname = self.confname + "-" + f = open(tname, 'wb') + # First output null subkey submap + if b'' in self.submaps: + for nm,value in self.submaps[b''].items(): + f.write(nm + b'=' + value + b'\n') + for sk,mp in self.submaps.items(): + if sk == b'': + continue + f.write(b'[' + sk + b']\n') + for nm,value in mp.items(): + f.write(nm + b'=' + value + b'\n') + f.close() + os.rename(tname, self.confname) + + def setbin(self, nm, value, sk = b''): + if self.readonly: + raise Exception("ConfSimple is readonly") + if sk not in self.submaps: + self.submaps[sk] = {} + self.submaps[sk][nm] = value + self._rewrite() + return True + + def set(self, nm, value, sk = b''): + if self.readonly: + raise Exception("ConfSimple is readonly") + if type(nm) == type(u''): + nm = nm.encode('utf-8') + if type(value) == type(u''): + value = value.encode('utf-8') + if type(sk) == type(u''): + sk = sk.encode('utf-8') + return self.setbin(nm, value, sk) + + +class ConfTree(ConfSimple): + """A ConfTree adds path-hierarchical interpretation of the section keys, + which should be '/'-separated values. When a value is requested for a + given path, it will also be searched in the sections corresponding to + the ancestors. E.g. get(name, '/a/b') will also look in sections '/a' and + '/' or '' (the last 2 are equivalent)""" + + def getbin(self, nm, sk = b''): + if type(nm) != type(b'') or type(sk) != type(b''): + raise TypeError("getbin: parameters must be binary not unicode") + #_debug("ConfTree::getbin: nm [%s] sk [%s]" % (nm, sk)) + + if sk == b'' or sk[0] != b'/'[0]: + return ConfSimple.getbin(self, nm, sk) + + if sk[len(sk)-1] == b'/'[0]: + sk = sk[:len(sk)-1] + + # Try all sk ancestors as submaps (/a/b/c-> /a/b/c, /a/b, /a, b'') + while sk: + if sk in self.submaps: + return ConfSimple.getbin(self, nm, sk) + if sk + b'/' in self.submaps: + return ConfSimple.getbin(self, nm, sk+b'/') + i = sk.rfind(b'/') + if i == -1: + break + sk = sk[:i] + + return ConfSimple.getbin(self, nm) + + +class ConfStack(object): + """ A ConfStack manages the superposition of a list of Configuration + objects. Values are looked for in each object from the list until found. + This typically provides for defaults overriden by sparse values in the + topmost file.""" + + def __init__(self, nm, dirs, tp = 'simple'): + fnames = [] + for dir in dirs: + fnm = os.path.join(dir, nm) + fnames.append(fnm) + self._construct(tp, fnames) + + def _construct(self, tp, fnames): + self.confs = [] + for fname in fnames: + if tp.lower() == 'simple': + conf = ConfSimple(fname) + else: + conf = ConfTree(fname) + self.confs.append(conf) + + # Accepts / returns binary strings (non-unicode) + def getbin(self, nm, sk = b''): + if type(nm) != type(b'') or type(sk) != type(b''): + raise TypeError("getbin: parameters must be binary not unicode") + for conf in self.confs: + value = conf.getbin(nm, sk) + if value is not None: + return value + return None + + def get(self, nm, sk = b''): + dodecode = False + if type(nm) == type(u''): + dodecode = True + nm = nm.encode('utf-8') + if type(sk) == type(u''): + sk = sk.encode('utf-8') + #v = ConfSimple.getbin(self, nm, sk) + v = self.getbin(nm, sk) + if v and dodecode: + v = v.decode('utf-8') + return v diff --git a/src/python/recoll/recoll/rclconfig.py b/src/python/recoll/recoll/rclconfig.py old mode 100755 new mode 100644 index 2e11b3ee..f680fe10 --- a/src/python/recoll/recoll/rclconfig.py +++ b/src/python/recoll/recoll/rclconfig.py @@ -8,129 +8,7 @@ import sys import base64 import platform -class ConfSimple: - """A ConfSimple class reads a recoll configuration file, which is a typical - ini file (see the Recoll manual). It's a dictionary of dictionaries which - lets you retrieve named values from the top level or a subsection""" - - def __init__(self, confname, tildexp = False): - self.submaps = {} - self.dotildexpand = tildexp - try: - f = open(confname, 'r') - except Exception as exc: - #print("Open Exception: %s" % exc, sys.stderr) - # File does not exist -> empty config, not an error. - return - - self.parseinput(f) - - def parseinput(self, f): - appending = False - line = '' - submapkey = '' - for cline in f: - cline = cline.rstrip("\r\n") - if appending: - line = line + cline - else: - line = cline - line = line.strip() - if line == '' or line[0] == '#': - continue - - if line[len(line)-1] == '\\': - line = line[0:len(line)-1] - appending = True - continue - appending = False - #print(line) - if line[0] == '[': - line = line.strip('[]') - if self.dotildexpand: - submapkey = os.path.expanduser(line) - else: - submapkey = line - #print("Submapkey: [%s]" % submapkey) - continue - nm, sep, value = line.partition('=') - if sep == '': - continue - nm = nm.strip() - value = value.strip() - #print("Name: [%s] Value: [%s]" % (nm, value)) - - if not submapkey in self.submaps: - self.submaps[submapkey] = {} - self.submaps[submapkey][nm] = value - - def get(self, nm, sk = ''): - '''Returns None if not found, empty string if found empty''' - if not sk in self.submaps: - return None - if not nm in self.submaps[sk]: - return None - return self.submaps[sk][nm] - - def getNames(self, sk = ''): - if not sk in self.submaps: - return None - return list(self.submaps[sk].keys()) - -class ConfTree(ConfSimple): - """A ConfTree adds path-hierarchical interpretation of the section keys, - which should be '/'-separated values. When a value is requested for a - given path, it will also be searched in the sections corresponding to - the ancestors. E.g. get(name, '/a/b') will also look in sections '/a' and - '/' or '' (the last 2 are equivalent)""" - def get(self, nm, sk = ''): - if sk == '' or sk[0] != '/': - return ConfSimple.get(self, nm, sk) - - if sk[len(sk)-1] != '/': - sk = sk + '/' - - # Try all sk ancestors as submaps (/a/b/c-> /a/b/c, /a/b, /a, '') - while sk.find('/') != -1: - val = ConfSimple.get(self, nm, sk) - if val is not None: - return val - i = sk.rfind('/') - if i == -1: - break - sk = sk[:i] - - return ConfSimple.get(self, nm) - -class ConfStack: - """ A ConfStack manages the superposition of a list of Configuration - objects. Values are looked for in each object from the list until found. - This typically provides for defaults overriden by sparse values in the - topmost file.""" - - def __init__(self, nm, dirs, tp = 'simple'): - fnames = [] - for dir in dirs: - fnm = os.path.join(dir, nm) - fnames.append(fnm) - self._construct(tp, fnames) - - def _construct(self, tp, fnames): - self.confs = [] - for fname in fnames: - if tp.lower() == 'simple': - conf = ConfSimple(fname) - else: - conf = ConfTree(fname) - self.confs.append(conf) - - def get(self, nm, sk = ''): - for conf in self.confs: - value = conf.get(nm, sk) - if value is not None: - return value - return None - +import conftree class RclDynConf: def __init__(self, fname): self.data = ConfSimple(fname) @@ -202,7 +80,7 @@ class RclConfig: def getConfParam(self, nm): if not self.config: - self.config = ConfStack("recoll.conf", self.cdirs, "tree") + self.config = conftree.ConfStack("recoll.conf", self.cdirs, "tree") return self.config.get(nm, self.keydir) class RclExtraDbs: