OwlCyberSecurity - MANAGER

Edit File: _pvector.py

from abc import abstractmethod, ABCMeta
from collections.abc import Sequence, Hashable
from numbers import Integral
import operator
from pyrsistent._transformations import transform

def _bitcount(val):
    return bin(val).count("1")

BRANCH_FACTOR = 32
BIT_MASK = BRANCH_FACTOR - 1
SHIFT = _bitcount(BIT_MASK)

def compare_pvector(v, other, operator):
    return operator(v.tolist(), other.tolist() if isinstance(other, PVector) else other)

def _index_or_slice(index, stop):
    if stop is None:
        return index

return slice(index, stop)

class PythonPVector(object):
    """
    Support structure for PVector that implements structural sharing for vectors using a trie.
    """
    __slots__ = ('_count', '_shift', '_root', '_tail', '_tail_offset', '__weakref__')

def __new__(cls, count, shift, root, tail):
        self = super(PythonPVector, cls).__new__(cls)
        self._count = count
        self._shift = shift
        self._root = root
        self._tail = tail

# Derived attribute stored for performance
        self._tail_offset = self._count - len(self._tail)
        return self

def __len__(self):
        return self._count

def __getitem__(self, index):
        if isinstance(index, slice):
            # There are more conditions than the below where it would be OK to
            # return ourselves, implement those...
            if index.start is None and index.stop is None and index.step is None:
                return self

# This is a bit nasty realizing the whole structure as a list before
            # slicing it but it is the fastest way I've found to date, and it's easy :-)
            return _EMPTY_PVECTOR.extend(self.tolist()[index])

if index < 0:
            index += self._count

return PythonPVector._node_for(self, index)[index & BIT_MASK]

def __add__(self, other):
        return self.extend(other)

def __repr__(self):
        return 'pvector({0})'.format(str(self.tolist()))

def __str__(self):
        return self.__repr__()

def __iter__(self):
        # This is kind of lazy and will produce some memory overhead but it is the fasted method
        # by far of those tried since it uses the speed of the built in python list directly.
        return iter(self.tolist())

def __ne__(self, other):
        return not self.__eq__(other)

def __eq__(self, other):
        return self is other or (hasattr(other, '__len__') and self._count == len(other)) and compare_pvector(self, other, operator.eq)

def __gt__(self, other):
        return compare_pvector(self, other, operator.gt)

def __lt__(self, other):
        return compare_pvector(self, other, operator.lt)

def __ge__(self, other):
        return compare_pvector(self, other, operator.ge)

def __le__(self, other):
        return compare_pvector(self, other, operator.le)

def __mul__(self, times):
        if times <= 0 or self is _EMPTY_PVECTOR:
            return _EMPTY_PVECTOR

if times == 1:
            return self

return _EMPTY_PVECTOR.extend(times * self.tolist())

__rmul__ = __mul__

def _fill_list(self, node, shift, the_list):
        if shift:
            shift -= SHIFT
            for n in node:
                self._fill_list(n, shift, the_list)
        else:
            the_list.extend(node)

def tolist(self):
        """
        The fastest way to convert the vector into a python list.
        """
        the_list = []
        self._fill_list(self._root, self._shift, the_list)
        the_list.extend(self._tail)
        return the_list

def _totuple(self):
        """
        Returns the content as a python tuple.
        """
        return tuple(self.tolist())

def __hash__(self):
        # Taking the easy way out again...
        return hash(self._totuple())

def transform(self, *transformations):
        return transform(self, transformations)

def __reduce__(self):
        # Pickling support
        return pvector, (self.tolist(),)

def mset(self, *args):
        if len(args) % 2:
            raise TypeError("mset expected an even number of arguments")

evolver = self.evolver()
        for i in range(0, len(args), 2):
            evolver[args[i]] = args[i+1]

return evolver.persistent()

class Evolver(object):
        __slots__ = ('_count', '_shift', '_root', '_tail', '_tail_offset', '_dirty_nodes',
                     '_extra_tail', '_cached_leafs', '_orig_pvector')

def __init__(self, v):
            self._reset(v)

def __getitem__(self, index):
            if not isinstance(index, Integral):
                raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)

if index < 0:
                index += self._count + len(self._extra_tail)

if self._count <= index < self._count + len(self._extra_tail):
                return self._extra_tail[index - self._count]

return PythonPVector._node_for(self, index)[index & BIT_MASK]

def _reset(self, v):
            self._count = v._count
            self._shift = v._shift
            self._root = v._root
            self._tail = v._tail
            self._tail_offset = v._tail_offset
            self._dirty_nodes = {}
            self._cached_leafs = {}
            self._extra_tail = []
            self._orig_pvector = v

def append(self, element):
            self._extra_tail.append(element)
            return self

def extend(self, iterable):
            self._extra_tail.extend(iterable)
            return self

def set(self, index, val):
            self[index] = val
            return self

def __setitem__(self, index, val):
            if not isinstance(index, Integral):
                raise TypeError("'%s' object cannot be interpreted as an index" % type(index).__name__)

if index < 0:
                index += self._count + len(self._extra_tail)

if 0 <= index < self._count:
                node = self._cached_leafs.get(index >> SHIFT)
                if node:
                    node[index & BIT_MASK] = val
                elif index >= self._tail_offset:
                    if id(self._tail) not in self._dirty_nodes:
                        self._tail = list(self._tail)
                        self._dirty_nodes[id(self._tail)] = True
                        self._cached_leafs[index >> SHIFT] = self._tail
                    self._tail[index & BIT_MASK] = val
                else:
                    self._root = self._do_set(self._shift, self._root, index, val)
            elif self._count <= index < self._count + len(self._extra_tail):
                self._extra_tail[index - self._count] = val
            elif index == self._count + len(self._extra_tail):
                self._extra_tail.append(val)
            else:
                raise IndexError("Index out of range: %s" % (index,))

def _do_set(self, level, node, i, val):
            if id(node) in self._dirty_nodes:
                ret = node
            else:
                ret = list(node)
                self._dirty_nodes[id(ret)] = True

if level == 0:
                ret[i & BIT_MASK] = val
                self._cached_leafs[i >> SHIFT] = ret
            else:
                sub_index = (i >> level) & BIT_MASK  # >>>
                ret[sub_index] = self._do_set(level - SHIFT, node[sub_index], i, val)

return ret

def delete(self, index):
            del self[index]
            return self

def __delitem__(self, key):
            if self._orig_pvector:
                # All structural sharing bets are off, base evolver on _extra_tail only
                l = PythonPVector(self._count, self._shift, self._root, self._tail).tolist()
                l.extend(self._extra_tail)
                self._reset(_EMPTY_PVECTOR)
                self._extra_tail = l

del self._extra_tail[key]

def persistent(self):
            result = self._orig_pvector
            if self.is_dirty():
                result = PythonPVector(self._count, self._shift, self._root, self._tail).extend(self._extra_tail)
                self._reset(result)

return result

def __len__(self):
            return self._count + len(self._extra_tail)

def is_dirty(self):
            return bool(self._dirty_nodes or self._extra_tail)

def evolver(self):
        return PythonPVector.Evolver(self)

def set(self, i, val):
        # This method could be implemented by a call to mset() but doing so would cause
        # a ~5 X performance penalty on PyPy (considered the primary platform for this implementation
        #  of PVector) so we're keeping this implementation for now.

if not isinstance(i, Integral):
            raise TypeError("'%s' object cannot be interpreted as an index" % type(i).__name__)

if i < 0:
            i += self._count

if 0 <= i < self._count:
            if i >= self._tail_offset:
                new_tail = list(self._tail)
                new_tail[i & BIT_MASK] = val
                return PythonPVector(self._count, self._shift, self._root, new_tail)

return PythonPVector(self._count, self._shift, self._do_set(self._shift, self._root, i, val), self._tail)

if i == self._count:
            return self.append(val)

raise IndexError("Index out of range: %s" % (i,))

def _do_set(self, level, node, i, val):
        ret = list(node)
        if level == 0:
            ret[i & BIT_MASK] = val
        else:
            sub_index = (i >> level) & BIT_MASK  # >>>
            ret[sub_index] = self._do_set(level - SHIFT, node[sub_index], i, val)

return ret

@staticmethod
    def _node_for(pvector_like, i):
        if 0 <= i < pvector_like._count:
            if i >= pvector_like._tail_offset:
                return pvector_like._tail

node = pvector_like._root
            for level in range(pvector_like._shift, 0, -SHIFT):
                node = node[(i >> level) & BIT_MASK]  # >>>

return node

raise IndexError("Index out of range: %s" % (i,))

def _create_new_root(self):
        new_shift = self._shift

# Overflow root?
        if (self._count >> SHIFT) > (1 << self._shift): # >>>
            new_root = [self._root, self._new_path(self._shift, self._tail)]
            new_shift += SHIFT
        else:
            new_root = self._push_tail(self._shift, self._root, self._tail)

return new_root, new_shift

def append(self, val):
        if len(self._tail) < BRANCH_FACTOR:
            new_tail = list(self._tail)
            new_tail.append(val)
            return PythonPVector(self._count + 1, self._shift, self._root, new_tail)

# Full tail, push into tree
        new_root, new_shift = self._create_new_root()
        return PythonPVector(self._count + 1, new_shift, new_root, [val])

def _new_path(self, level, node):
        if level == 0:
            return node

return [self._new_path(level - SHIFT, node)]

def _mutating_insert_tail(self):
        self._root, self._shift = self._create_new_root()
        self._tail = []

def _mutating_fill_tail(self, offset, sequence):
        max_delta_len = BRANCH_FACTOR - len(self._tail)
        delta = sequence[offset:offset + max_delta_len]
        self._tail.extend(delta)
        delta_len = len(delta)
        self._count += delta_len
        return offset + delta_len

def _mutating_extend(self, sequence):
        offset = 0
        sequence_len = len(sequence)
        while offset < sequence_len:
            offset = self._mutating_fill_tail(offset, sequence)
            if len(self._tail) == BRANCH_FACTOR:
                self._mutating_insert_tail()

self._tail_offset = self._count - len(self._tail)

def extend(self, obj):
        # Mutates the new vector directly for efficiency but that's only an
        # implementation detail, once it is returned it should be considered immutable
        l = obj.tolist() if isinstance(obj, PythonPVector) else list(obj)
        if l:
            new_vector = self.append(l[0])
            new_vector._mutating_extend(l[1:])
            return new_vector

return self

def _push_tail(self, level, parent, tail_node):
        """
        if parent is leaf, insert node,
        else does it map to an existing child? ->
             node_to_insert = push node one more level
        else alloc new path

return  node_to_insert placed in copy of parent
        """
        ret = list(parent)

if level == SHIFT:
            ret.append(tail_node)
            return ret

sub_index = ((self._count - 1) >> level) & BIT_MASK  # >>>
        if len(parent) > sub_index:
            ret[sub_index] = self._push_tail(level - SHIFT, parent[sub_index], tail_node)
            return ret

ret.append(self._new_path(level - SHIFT, tail_node))
        return ret

def index(self, value, *args, **kwargs):
        return self.tolist().index(value, *args, **kwargs)

def count(self, value):
        return self.tolist().count(value)

def delete(self, index, stop=None):
        l = self.tolist()
        del l[_index_or_slice(index, stop)]
        return _EMPTY_PVECTOR.extend(l)

def remove(self, value):
        l = self.tolist()
        l.remove(value)
        return _EMPTY_PVECTOR.extend(l)

class PVector(metaclass=ABCMeta):
    """
    Persistent vector implementation. Meant as a replacement for the cases where you would normally
    use a Python list.

Do not instantiate directly, instead use the factory functions :py:func:`v` and :py:func:`pvector` to
    create an instance.

Heavily influenced by the persistent vector available in Clojure. Initially this was more or
    less just a port of the Java code for the Clojure vector. It has since been modified and to
    some extent optimized for usage in Python.

The vector is organized as a trie, any mutating method will return a new vector that contains the changes. No
    updates are done to the original vector. Structural sharing between vectors are applied where possible to save
    space and to avoid making complete copies.

This structure corresponds most closely to the built in list type and is intended as a replacement. Where the
    semantics are the same (more or less) the same function names have been used but for some cases it is not possible,
    for example assignments.

The PVector implements the Sequence protocol and is Hashable.

Inserts are amortized O(1). Random access is log32(n) where n is the size of the vector.

The following are examples of some common operations on persistent vectors:

>>> p = v(1, 2, 3)
    >>> p2 = p.append(4)
    >>> p3 = p2.extend([5, 6, 7])
    >>> p
    pvector([1, 2, 3])
    >>> p2
    pvector([1, 2, 3, 4])
    >>> p3
    pvector([1, 2, 3, 4, 5, 6, 7])
    >>> p3[5]
    6
    >>> p.set(1, 99)
    pvector([1, 99, 3])
    >>>
    """

@abstractmethod
    def __len__(self):
        """
        >>> len(v(1, 2, 3))
        3
        """

@abstractmethod
    def __getitem__(self, index):
        """
        Get value at index. Full slicing support.

>>> v1 = v(5, 6, 7, 8)
        >>> v1[2]
        7
        >>> v1[1:3]
        pvector([6, 7])
        """

@abstractmethod
    def __add__(self, other):
        """
        >>> v1 = v(1, 2)
        >>> v2 = v(3, 4)
        >>> v1 + v2
        pvector([1, 2, 3, 4])
        """

@abstractmethod
    def __mul__(self, times):
        """
        >>> v1 = v(1, 2)
        >>> 3 * v1
        pvector([1, 2, 1, 2, 1, 2])
        """

@abstractmethod
    def __hash__(self):
        """
        >>> v1 = v(1, 2, 3)
        >>> v2 = v(1, 2, 3)
        >>> hash(v1) == hash(v2)
        True
        """

@abstractmethod
    def evolver(self):
        """
        Create a new evolver for this pvector. The evolver acts as a mutable view of the vector
        with "transaction like" semantics. No part of the underlying vector i updated, it is still
        fully immutable. Furthermore multiple evolvers created from the same pvector do not
        interfere with each other.

You may want to use an evolver instead of working directly with the pvector in the
        following cases:

* Multiple updates are done to the same vector and the intermediate results are of no
          interest. In this case using an evolver may be a more efficient and easier to work with.
        * You need to pass a vector into a legacy function or a function that you have no control
          over which performs in place mutations of lists. In this case pass an evolver instance
          instead and then create a new pvector from the evolver once the function returns.

The following example illustrates a typical workflow when working with evolvers. It also
        displays most of the API (which i kept small by design, you should not be tempted to
        use evolvers in excess ;-)).

Create the evolver and perform various mutating updates to it:

>>> v1 = v(1, 2, 3, 4, 5)
        >>> e = v1.evolver()
        >>> e[1] = 22
        >>> _ = e.append(6)
        >>> _ = e.extend([7, 8, 9])
        >>> e[8] += 1
        >>> len(e)
        9

The underlying pvector remains the same:

>>> v1
        pvector([1, 2, 3, 4, 5])

The changes are kept in the evolver. An updated pvector can be created using the
        persistent() function on the evolver.

>>> v2 = e.persistent()
        >>> v2
        pvector([1, 22, 3, 4, 5, 6, 7, 8, 10])

The new pvector will share data with the original pvector in the same way that would have
        been done if only using operations on the pvector.
        """

@abstractmethod
    def mset(self, *args):
        """
        Return a new vector with elements in specified positions replaced by values (multi set).

Elements on even positions in the argument list are interpreted as indexes while
        elements on odd positions are considered values.

>>> v1 = v(1, 2, 3)
        >>> v1.mset(0, 11, 2, 33)
        pvector([11, 2, 33])
        """

@abstractmethod
    def set(self, i, val):
        """
        Return a new vector with element at position i replaced with val. The original vector remains unchanged.

Setting a value one step beyond the end of the vector is equal to appending. Setting beyond that will
        result in an IndexError.

>>> v1 = v(1, 2, 3)
        >>> v1.set(1, 4)
        pvector([1, 4, 3])
        >>> v1.set(3, 4)
        pvector([1, 2, 3, 4])
        >>> v1.set(-1, 4)
        pvector([1, 2, 4])
        """

@abstractmethod
    def append(self, val):
        """
        Return a new vector with val appended.

>>> v1 = v(1, 2)
        >>> v1.append(3)
        pvector([1, 2, 3])
        """

@abstractmethod
    def extend(self, obj):
        """
        Return a new vector with all values in obj appended to it. Obj may be another
        PVector or any other Iterable.

>>> v1 = v(1, 2, 3)
        >>> v1.extend([4, 5])
        pvector([1, 2, 3, 4, 5])
        """

@abstractmethod
    def index(self, value, *args, **kwargs):
        """
        Return first index of value. Additional indexes may be supplied to limit the search to a
        sub range of the vector.

>>> v1 = v(1, 2, 3, 4, 3)
        >>> v1.index(3)
        2
        >>> v1.index(3, 3, 5)
        4
        """

@abstractmethod
    def count(self, value):
        """
        Return the number of times that value appears in the vector.

>>> v1 = v(1, 4, 3, 4)
        >>> v1.count(4)
        2
        """

@abstractmethod
    def transform(self, *transformations):
        """
        Transform arbitrarily complex combinations of PVectors and PMaps. A transformation
        consists of two parts. One match expression that specifies which elements to transform
        and one transformation function that performs the actual transformation.

>>> from pyrsistent import freeze, ny
        >>> news_paper = freeze({'articles': [{'author': 'Sara', 'content': 'A short article'},
        ...                                   {'author': 'Steve', 'content': 'A slightly longer article'}],
        ...                      'weather': {'temperature': '11C', 'wind': '5m/s'}})
        >>> short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:25] + '...' if len(c) > 25 else c)
        >>> very_short_news = news_paper.transform(['articles', ny, 'content'], lambda c: c[:15] + '...' if len(c) > 15 else c)
        >>> very_short_news.articles[0].content
        'A short article'
        >>> very_short_news.articles[1].content
        'A slightly long...'

When nothing has been transformed the original data structure is kept

>>> short_news is news_paper
        True
        >>> very_short_news is news_paper
        False
        >>> very_short_news.articles[0] is news_paper.articles[0]
        True
        """

@abstractmethod
    def delete(self, index, stop=None):
        """
        Delete a portion of the vector by index or range.

>>> v1 = v(1, 2, 3, 4, 5)
        >>> v1.delete(1)
        pvector([1, 3, 4, 5])
        >>> v1.delete(1, 3)
        pvector([1, 4, 5])
        """

@abstractmethod
    def remove(self, value):
        """
        Remove the first occurrence of a value from the vector.

>>> v1 = v(1, 2, 3, 2, 1)
        >>> v2 = v1.remove(1)
        >>> v2
        pvector([2, 3, 2, 1])
        >>> v2.remove(1)
        pvector([2, 3, 2])
        """

_EMPTY_PVECTOR = PythonPVector(0, SHIFT, [], [])
PVector.register(PythonPVector)
Sequence.register(PVector)
Hashable.register(PVector)

def python_pvector(iterable=()):
    """
    Create a new persistent vector containing the elements in iterable.

>>> v1 = pvector([1, 2, 3])
    >>> v1
    pvector([1, 2, 3])
    """
    return _EMPTY_PVECTOR.extend(iterable)

try:
    # Use the C extension as underlying trie implementation if it is available
    import os
    if os.environ.get('PYRSISTENT_NO_C_EXTENSION'):
        pvector = python_pvector
    else:
        from pvectorc import pvector
        PVector.register(type(pvector()))
except ImportError:
    pvector = python_pvector

def v(*elements):
    """
    Create a new persistent vector containing all parameters to this function.

>>> v1 = v(1, 2, 3)
    >>> v1
    pvector([1, 2, 3])
    """
    return pvector(elements)