doc.py

#!/usr/bin/env python -3
# encoding: utf-8
#Created by  on Wed Oct 20 11:41:16 CDT 2010

# Copyright (C) 2010 Graham I Cummins
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA
#

#ClassCase, functionCase, datacase

import search
import re
from gicdat.control import report
from threading import Lock

LINK = search.pattern({'_link': 's'})
SLICE = search.pattern({'_link': 's', '_slice': '['})
EXTERN = search.pattern({'_extern_url': 's'})


def ishashable(o):
    try:
        _ = hash(o)
        return True
    except TypeError:
        return False


class DictAccessor(object):
    '''
    This class provides a mapping from item access ([]) to attribute access
    (.). It wraps an underlying Doc and allows the elements in that Doc to
    be accessed using d.k notation in place of d["k"]

    '''
    d = None

    def __init__(self, d):
        self._doc = d

    def __getattr__(self, k):
        v = self._doc[k]
        if type(v) == Doc:
            return DictAccessor(v)
        else:
            return v

    def keys(self, depth=0, skip=None, sort=False):
        return self._doc.keys(depth=0, skip=None, sort=False)

    def __str__(self):
        return "Doc Accessor: %s" % (str(self._doc),)

    def __repr__(self):
        return str(self)


trailingdigits = re.compile("^(.*?)(\d+)$")
TDIG = re.compile("(\d+)$")


def _cmptint(s1, s2):
    m1 = TDIG.search(s1)
    m2 = TDIG.search(s2)
    if m1 and m2:
        return cmp(int(m1.groups()[0]), int(m2.groups()[0]))
    else:
        return cmp(s1, s2)


def strinc(s):
    m = trailingdigits.match(s)
    if not m:
        return s + "1"
    else:
        b, i = m.groups()
        return b + str(int(i) + 1)


def astuple(a):
    if hasattr(a, '__iter__'):
        return tuple([astuple(i) for i in a])
    else:
        return a


class Doc(dict):
    def __init__(self, arg=None, safe=1):
        '''
        arg:<can be passed to dict>, safe: bool, up: None | (Doc, str) -> r: Doc
        Create a new instance. Arg should be something that can be passed to the
        built-in "dict" function. If safe, then keys are passed through self.safekey,
        and values through self.safevalue

        '''
        dict.__init__(self)
        if arg:
            d = dict(arg)
            self.update(d, safe, False)

    def new(self, **kwargs):
        '''
        returns an empty Doc instance, or, if there are keyword arguments, the
        Document generated by calling Doc(kwargs)  (where kwargs is the dictionary
        of all keyword arguments passed
        '''
        return Doc(kwargs)

    def safekey(self, k, stripdots=False, unique=False):
        '''
        k:any -> r:str  Returns a string based on k that is a legitimate key
        (currently that is just  str(k) ). If stripdots is True, replace any "."
        characters with the search.DOT pattern (thus assuring that the key will be
        assigned to the local dict, not  to a nested one).If unique is True, then if
        the return value would be an existing key, it is modified. If it ends in
        digits, the integer represented by these digits is incremented until the
        result is not a key. If it doesn't end in digits, a "1" is appended (and
        incremented if needed).

        '''
        k = str(k)
        if stripdots:
            k = k.replace('.', search.DOT)
        if unique and self.get(k) != None:
            return self.safekey(strinc(k), False, True)
        else:
            return k

    def safeval(self, v):
        '''
        v: object -> r: None|tuple|search.SIMPLE|search.ARRAY
        convert v to a "safe" type, which is either a simple type,
        a hashable sequence, or an ndarray instance. In
        the latter case, the array will be set read-only.

        A TypeError is thrown if the input is not one of these types and also
        can't be converted to one (for example, lists can usually be cast to
        tuples,but functions, classes, etc will usually not convert)

        '''
        vt = search.classify(v)
        if vt in search.simpletypes:
            return v
        elif vt == '#':
            v.setflags(write=False)
            return v
        elif vt == '[':
            if ishashable(v):
                return v
            return astuple(v)
        elif vt == "{":
            return Doc(v, 2)
        else:
            raise TypeError('type of %s (%s) is not safe (and can not be made so)' % (str(v), str(type(v)),))

    def __getitem__(self, k):
        return self.get(k)

    def __setitem__(self, k, v):
        return self.set(k, v)

    def __delitem__(self, k):
        self.set(k, None)

    def clean(self):
        for k in self.keys():
            v = dict.__getitem__(self, k)
            if v == None:
                dict.__delitem__(self, k)
            elif type(v) == Doc:
                v.clean()

    def keys(self, depth=0, match=None, sort=False, subdockeys=True):
        if not depth and not match and subdockeys:
            keys = dict.keys(self)
        else:
            keys = [p for p in self.iter(depth, match, subdockeys)]
        if sort:
            if not callable(sort):
                sort = _cmptint
            keys = sorted(keys, sort)
        return keys

    def __str__(self):
        s = ["{"]
        for k in self.keys():
            s.append('%s:%s' % (k, self.summary(k)))
        s.append('}')
        return "\n".join(s)

    def summary(self, k):
        v = self[k]
        vt = search.classify(v)
        if vt in search.simpletypes:
            return str(v)
        elif vt == '#':
            if v.size < 10:
                return str(v)
            else:
                return "%s-array:%s" % (str(v.shape), v.dtype.str)
        elif vt == '[':
            if len(str(v)) < 60:
                return str(v)
            else:
                return search.seqtype(v)
        elif vt == '{':
            n = len(v.keys())
            sn = len(v.keys(depth=-1)) - n
            return "%i-doc (%i subkeys)" % (n, sn)
        else:
            return str(v)

    def iter(self, depth=-1, match=None, subdockeys=True):
        '''
        match is the string content for a regular expression, which allows the
        return of only keys that match it (using re.match, not re.search, and
        extended in dot notation!)

        if subdockeys is False, then keys associated to sub-documents are not
        returned however, sub-keys within these documents are still returned unless
        the subdocuments match a skip predicate.

        subdockeys may also be "only" which allows only these keys to be returned
        '''
        its = []
        its.insert(0, (self.__iter__(), self, ''))
        if match:
            match = re.compile(match)
        while True:
            try:
                k = its[0][0].next()
            except StopIteration:
                its.pop(0)
                if its:
                    continue
                else:
                    raise
            if its[0][2]:
                k = its[0][2] + '.' + k
            m = True
            res = False
            if match:
                m = match.match(k)
            if m and subdockeys != True:
                m = not type(self[k]) == Doc
                if subdockeys == "only":
                    m = not m
            if m:
                res = (yield k)
            if not res and (depth < 0 or depth >= len(its)):
                v = self[k]
                if type(v) == Doc:
                    its.insert(0, (v.__iter__(), v, k))

    def set(self, k, v, safe=1, unique=False, fast=False):
        '''
        Assigns a value v to a key k in self in some manner.

        If fast is True, this calls dict.__setitem__. Use this for better performance
        if you know that your key and value are correct (key is a string containing
        no "." characters, v is a safe value (see self.safeval) ).

        Otherwise set checks the key for "." separation and assigns to sub-documents
        if needed. Intermediate sub-documents are created as needed.

        If unique is True, k is converted by self.safekey(k, False, True)

        If safe is True, v is converted by self.safeval(v), and k is
        converted by self.safekey(k, False, False).


        '''
        if fast:
            return dict.__setitem__(self, k, v)
        if safe or unique:
            k = self.safekey(k, False, unique)
        if safe > 1:
            v = self.safeval(v)
        elif safe and type(v) == dict:
            v = Doc(v)

        if not '.' in k:
            return dict.__setitem__(self, k, v)
        else:
            k = k.split('.')
            k0 = self.get(k[0])
            if type(k0) != Doc:
                k0 = Doc()
                self.set(k[0], k0, fast=True)
            return self[k[0]].set('.'.join(k[1:]), v, False, False)

    def resolvelink(self, d, k='', value=True):
        '''
        return the value of a link or slice dictionary d, located at key k
        (if k is None, relative links are not allowed)
        '''
        k = k.split('.')
        t = d['_link']
        if t.startswith('.'):
            nt = t.rstrip('.')
            nl = len(t) - len(nt)
            if nl > len(k):
                report('WARNING: relative link %s encountered at %s. Not enough nesting levels to specify it.' % (
                t, '.'.join(k)))
                return None
            else:
                t = '.'.join(k[:-nl]) + "." + nt
        if not value:
            return t
        if not t.strip():
            return self
        nv = self.get(t)
        if nv == None:
            report('WARNING: link at %s to %s is broken' % ('.'.join(k), t))
            return None
        if d['_slice'] != None:
            nvt = search.classify(nv)
            sl = d['_slice']
            if nvt == '#':
                if not type(sl[0]) == tuple:
                    sl = (sl,)
                nv = nv[[apply(slice, t) for t in sl]]
            elif nvt == '[':
                if type(sl[0]) == tuple:
                    sl = sl[0]
                nv = nv[apply(slice, sl)]
            else:
                report('WARNING: slice at %s of %s tries to slice a non-sequence' % ('.'.join(k), t))
                nv = None
        return nv

    def get(self, k, follow=False, fast=False):
        '''

        If fast is True, this calls dict.get(k). Use this is you know you just
        need simple access to a local key.

        Otherwise, k is split on '.' characters, and values are retrieved from
        sub-documents.

        If follow is True, then slice and link Docs will not be returned,
        instead the item or slice that they represent will be returned

        If root is not None, then it is used to resolve link following.
        Typically this argument is used only by recursive calls.

        '''
        if fast:
            return dict.get(self, k)
        if isinstance(k, dict):
            return self.resolvelink(k)
        if not '.' in k:
            v = dict.get(self, k)
            k = [k]
        else:
            k = k.split('.')
            sd = self.get(k[0])
            if sd:
                v = sd.get('.'.join(k[1:]))
            else:
                v = None
        if follow and LINK == v:
            return self.resolvelink(v, k)
        return v

    def update(self, d, safe=False, unique=False):
        ''' calls self.set(k, d[k], safe, unique) for k in d '''
        if type(d) == Doc:
            keys = d.keys(depth=-1, subdockeys=False)
        else:
            keys = d.keys()
        for k in keys:
            self.set(k, d[k], safe, unique)

    def find(self, p="_", depth=-1, skip=None, keys=(),
             tops=False, setns=False):
        '''
        p is an argument to search.pattern,
        depth: int, skip: function, keys: (str...) -> r: generator

        this is a generator over keys k such that p == self[k]

        if keys is specified, then depth and skip are ignored, and the search
        is performed only over keys. Otherwise, the search is performed over
        self.iter(depth, skip)

        if tops is true, and p can match Docs, then the subkeys of any subdoc that
        matches p will not be searched. This will only work as expected if "keys"
        is not set.

        If setns is True, then p should be a search.Pattern instance. This function
        will assign self to the key "doc" in p.ns, and call p.setup before searching.

        Note that this is a generator, not a function that returns a collection.
        To get the list of hits, use [k for k in self.find(...)]

        (this idiom is bound to self.findall)

        To get the first hit, use:
        try:
            v = self.find.next()
        except StopIteration:
            #nothing found
            v = None

        '''
        if keys:
            keys = keys.__iter__()
        else:
            keys = self.iter(depth, skip)
        p = search.pattern(p)
        if setns:
            p.ns['doc'] = self
            p.setup()
        v = False
        while True:
            if v and tops:
                k = keys.send(True)
            else:
                k = keys.next()
            v = bool(p == self.get(k))
            if v:
                (yield k)

    def findall(self, p, depth=-1, skip=None, keys=(), tops=False, setns=False):
        '''
        This is a shortcut into self.find, which returns a list rather than an iterator.
        See Doc.find documentation to explain the arguments.
        '''
        return [k for k in self.find(p, depth=-1, skip=None, keys=(), tops=False, setns=False)]


    @property
    def d(self):
        return DictAccessor(self)

    def setslice(self, k, sl, v):
        if not type(sl[0]) in [tuple, list]:
            sl = apply(slice, sl)
        else:
            sl = tuple([apply(slice, t) for t in sl])
        ov = self[k]
        ovt = search.classify(ov)
        if ovt == '#':
            ov[sl] = v
        elif ovt == '[':
            nv = list(ov)
            nv[sl] = v
            self[k] = tuple(nv)
        else:
            raise ValueError('Attempt to assign slice to non-sequence at %s' % (k,))

    def patch(self, d, inverse=False):
        lock = None
        if inverse:
            inverse = Doc()
        for k in d.keys(-1, subdockeys=False):
            if k.startswith("->"):
                lk = k[2:]
                t = self[lk + '._link']
                if t == None:
                    raise KeyError("Attempt to set the target of link at %s, but it is not a link" % k)
                s = self[lk + '._slice']
                if s == None:
                    if inverse != False:
                        inverse[t] = self[t]
                    self[t] = d[k]
                else:
                    if inverse != False:
                        inverse[lk] = self.resolvelink(self[lk])
                    self.setslice(self[lk], d[k])
            else:
                if inverse != False:
                    inverse[k] = self[k]
                self[k] = d[k]
        self.clean()
        return inverse

    def fuse(self, *docs):
        '''
        return a new doc that is made by creating a copy of self, and then
        patching it with each document in the argument list. Updates occur in order,
        and without "unique", so values in later arguments overwrite those in earlier arguments.

        Arguments that are dictionaries are cast to gd.Doc

        no inverse patches are generated. The return value is the final document instance


        '''
        nd = self.copy()
        for d in docs:
            if type(d) == dict:
                d = Doc(d)
            nd.patch(d, False)
        return nd

    def copy(self):
        '''
        returns a new doc with the same keys (recursively) as self.
        This is a deep copy, in the sense that all sub-documents are replaced with new Docs. The actual
        values are not copied, but this should be OK, since these values should all be immutable. If you
        use evil code that end-runs around stored arrays setting the write flag false, and modifies these
        arrays directly, you will get in trouble with side effects, but it is no more than you deserve.

        '''
        d = Doc()
        d.update(self, False, False)
        return d

    def subset(self, keys, links='include'):
        d = Doc()
        for k in keys:
            if links and LINK == self[k]:
                if links == 'include':
                    lt = self.resolvelink(k, self[k]['_link'], False)
                    d[lt] = self[lt]
                elif links == 'follow':
                    d[k] = self.resolvelink(k, self[k]['_link'])
                else:
                    raise ValueError('links switch should be "include", "follow", or False')
            else:
                d.set(k, self[k], safe=False)
        return d

    def inc_counter(self, k, n=1, sv=0):
        '''
        Increase the value stored in key k by n (default 1). These values should be
        numbers. If there is no value at key k, use sv as the starting value (default
        0).

        '''
        if self[k] == None:
            self[k] = sv + n
        else:
            self[k] = self[k] + n

    def appendtokey(self, k, v, unique=False):
        '''
        If there is no value at self[k], set self[k]= (v,). Otherwise self[k] should be a tuple,
        and is updateds to self[k] = self[k] + (v,). If unique is true, the update occurs only
        if v is not already in self[k]

        '''
        if self[k] == None:
            self[k] = (v,)
        elif not unique or not v in self[k]:
            self[k] = self[k] + (v,)

    def astuple(self):
        t = []
        for k in self.keys(-1, None, True, False):
            v = self[k]
            if v == None:
                continue
            if search.classify(v) in "[#":
                v = astuple(v)
            t.append((k, v))
        return tuple(t)

    def hash(self):
        return hash(self.astuple())