Merge branch 'develop' into dev_graphviz
This commit is contained in:
Romain Dorgueil
6
bonobo/util/collections.py
Normal file
6
bonobo/util/collections.py
Normal file
@ -0,0 +1,6 @@
|
||||
import bisect
|
||||
|
||||
|
||||
class sortedlist(list):
|
||||
def insort(self, x):
|
||||
bisect.insort(self, x)
|
||||
@ -1,31 +1,7 @@
|
||||
import functools
|
||||
import struct
|
||||
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
|
||||
def is_platform_little_endian():
|
||||
""" am I little endian """
|
||||
return sys.byteorder == 'little'
|
||||
|
||||
|
||||
def is_platform_windows():
|
||||
return sys.platform == 'win32' or sys.platform == 'cygwin'
|
||||
|
||||
|
||||
def is_platform_linux():
|
||||
return sys.platform == 'linux2'
|
||||
|
||||
|
||||
def is_platform_mac():
|
||||
return sys.platform == 'darwin'
|
||||
|
||||
|
||||
def is_platform_32bit():
|
||||
return struct.calcsize("P") * 8 < 64
|
||||
|
||||
|
||||
def deprecated_alias(alias, func):
|
||||
@functools.wraps(func)
|
||||
def new_func(*args, **kwargs):
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
import sys
|
||||
from textwrap import indent
|
||||
|
||||
from bonobo import settings
|
||||
from bonobo.structs.bags import ErrorBag
|
||||
|
||||
|
||||
@ -7,7 +9,14 @@ def is_error(bag):
|
||||
return isinstance(bag, ErrorBag)
|
||||
|
||||
|
||||
def print_error(exc, trace, context=None, prefix=''):
|
||||
def _get_error_message(exc):
|
||||
if hasattr(exc, '__str__'):
|
||||
message = str(exc)
|
||||
return message[0].upper() + message[1:]
|
||||
return '\n'.join(exc.args),
|
||||
|
||||
|
||||
def print_error(exc, trace, context=None, method=None):
|
||||
"""
|
||||
Error handler. Whatever happens in a plugin or component, if it looks like an exception, taste like an exception
|
||||
or somehow make me think it is an exception, I'll handle it.
|
||||
@ -18,14 +27,20 @@ def print_error(exc, trace, context=None, prefix=''):
|
||||
"""
|
||||
|
||||
from colorama import Fore, Style
|
||||
|
||||
prefix = '{}{} | {}'.format(Fore.RED, Style.BRIGHT, Style.RESET_ALL)
|
||||
|
||||
print(
|
||||
Style.BRIGHT,
|
||||
Fore.RED,
|
||||
'\U0001F4A3 {}{}{}'.format(
|
||||
(prefix + ': ') if prefix else '', type(exc).__name__, ' in {!r}'.format(context) if context else ''
|
||||
),
|
||||
type(exc).__name__,
|
||||
' (in {}{})'.format(type(context).__name__, '.{}()'.format(method) if method else '') if context else '',
|
||||
Style.RESET_ALL,
|
||||
'\n',
|
||||
indent(_get_error_message(exc), prefix + Style.BRIGHT),
|
||||
Style.RESET_ALL,
|
||||
sep='',
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(trace)
|
||||
print(prefix, file=sys.stderr)
|
||||
print(indent(trace, prefix, predicate=lambda line: True), file=sys.stderr)
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
from bonobo.util.compat import deprecated
|
||||
|
||||
|
||||
@deprecated
|
||||
def console_run(*chain, output=True, plugins=None, strategy=None):
|
||||
from bonobo import run
|
||||
from bonobo.ext.console import ConsoleOutputPlugin
|
||||
|
||||
return run(*chain, plugins=(plugins or []) + [ConsoleOutputPlugin()] if output else [], strategy=strategy)
|
||||
|
||||
|
||||
@deprecated
|
||||
def jupyter_run(*chain, plugins=None, strategy=None):
|
||||
from bonobo import run
|
||||
from bonobo.ext.jupyter import JupyterOutputPlugin
|
||||
|
||||
return run(*chain, plugins=(plugins or []) + [JupyterOutputPlugin()], strategy=strategy)
|
||||
116
bonobo/util/inspect.py
Normal file
116
bonobo/util/inspect.py
Normal file
@ -0,0 +1,116 @@
|
||||
from collections import namedtuple
|
||||
|
||||
|
||||
def isconfigurabletype(mixed):
|
||||
"""
|
||||
Check if the given argument is an instance of :class:`bonobo.config.ConfigurableMeta`, meaning it has all the
|
||||
plumbery necessary to build :class:`bonobo.config.Configurable`-like instances.
|
||||
|
||||
:param mixed:
|
||||
:return: bool
|
||||
"""
|
||||
from bonobo.config.configurables import ConfigurableMeta
|
||||
return isinstance(mixed, ConfigurableMeta)
|
||||
|
||||
|
||||
def isconfigurable(mixed):
|
||||
"""
|
||||
Check if the given argument is an instance of :class:`bonobo.config.Configurable`.
|
||||
|
||||
:param mixed:
|
||||
:return: bool
|
||||
"""
|
||||
from bonobo.config.configurables import Configurable
|
||||
return isinstance(mixed, Configurable)
|
||||
|
||||
|
||||
def isoption(mixed):
|
||||
"""
|
||||
Check if the given argument is an instance of :class:`bonobo.config.Option`.
|
||||
|
||||
:param mixed:
|
||||
:return: bool
|
||||
"""
|
||||
|
||||
from bonobo.config.options import Option
|
||||
return isinstance(mixed, Option)
|
||||
|
||||
|
||||
def ismethod(mixed):
|
||||
"""
|
||||
Check if the given argument is an instance of :class:`bonobo.config.Method`.
|
||||
|
||||
:param mixed:
|
||||
:return: bool
|
||||
"""
|
||||
from bonobo.config.options import Method
|
||||
return isinstance(mixed, Method)
|
||||
|
||||
|
||||
def iscontextprocessor(x):
|
||||
"""
|
||||
Check if the given argument is an instance of :class:`bonobo.config.ContextProcessor`.
|
||||
|
||||
:param mixed:
|
||||
:return: bool
|
||||
"""
|
||||
from bonobo.config.processors import ContextProcessor
|
||||
return isinstance(x, ContextProcessor)
|
||||
|
||||
|
||||
def istype(mixed):
|
||||
"""
|
||||
Check if the given argument is a type object.
|
||||
|
||||
:param mixed:
|
||||
:return: bool
|
||||
"""
|
||||
return isinstance(mixed, type)
|
||||
|
||||
|
||||
ConfigurableInspection = namedtuple(
|
||||
'ConfigurableInspection', [
|
||||
'type',
|
||||
'instance',
|
||||
'options',
|
||||
'processors',
|
||||
'partial',
|
||||
]
|
||||
)
|
||||
|
||||
ConfigurableInspection.__enter__ = lambda self: self
|
||||
ConfigurableInspection.__exit__ = lambda *exc_details: None
|
||||
|
||||
|
||||
def inspect_node(mixed, *, _partial=None):
|
||||
"""
|
||||
If the given argument is somehow a :class:`bonobo.config.Configurable` object (either a subclass, an instance, or
|
||||
a partially configured instance), then it will return a :class:`ConfigurableInspection` namedtuple, used to inspect
|
||||
the configurable metadata (options). If you want to get the option values, you don't need this, it is only usefull
|
||||
to perform introspection on a configurable.
|
||||
|
||||
If it's not looking like a configurable, it will raise a :class:`TypeError`.
|
||||
|
||||
:param mixed:
|
||||
:return: ConfigurableInspection
|
||||
|
||||
:raise: TypeError
|
||||
"""
|
||||
if isconfigurabletype(mixed):
|
||||
inst, typ = None, mixed
|
||||
elif isconfigurable(mixed):
|
||||
inst, typ = mixed, type(mixed)
|
||||
elif hasattr(mixed, 'func'):
|
||||
return inspect_node(mixed.func, _partial=(mixed.args, mixed.keywords))
|
||||
else:
|
||||
raise TypeError(
|
||||
'Not a Configurable, nor a Configurable instance and not even a partially configured Configurable. Check your inputs.'
|
||||
)
|
||||
|
||||
return ConfigurableInspection(
|
||||
typ,
|
||||
inst,
|
||||
list(typ.__options__),
|
||||
list(typ.__processors__),
|
||||
_partial,
|
||||
)
|
||||
@ -1,4 +1,5 @@
|
||||
""" Iterator utilities. """
|
||||
import functools
|
||||
|
||||
|
||||
def force_iterator(mixed):
|
||||
@ -23,6 +24,19 @@ def ensure_tuple(tuple_or_mixed):
|
||||
return (tuple_or_mixed, )
|
||||
|
||||
|
||||
def tuplize(generator):
|
||||
""" Takes a generator and make it a tuple-returning function. As a side
|
||||
effect, it can also decorate any iterator-returning function to force
|
||||
return value to be a tuple.
|
||||
"""
|
||||
|
||||
@functools.wraps(generator)
|
||||
def tuplized(*args, **kwargs):
|
||||
return tuple(generator(*args, **kwargs))
|
||||
|
||||
return tuplized
|
||||
|
||||
|
||||
def iter_if_not_sequence(mixed):
|
||||
if isinstance(mixed, (dict, list, str)):
|
||||
raise TypeError(type(mixed).__name__)
|
||||
|
||||
@ -1,3 +1,7 @@
|
||||
import functools
|
||||
from functools import partial
|
||||
|
||||
|
||||
def get_name(mixed):
|
||||
try:
|
||||
return mixed.__name__
|
||||
@ -27,178 +31,194 @@ class ValueHolder:
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, value, *, type=None):
|
||||
self.value = value
|
||||
self.type = type
|
||||
def __init__(self, value):
|
||||
self._value = value
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.value)
|
||||
@property
|
||||
def value(self):
|
||||
# XXX deprecated
|
||||
return self._value
|
||||
|
||||
def __lt__(self, other):
|
||||
return self.value < other
|
||||
def get(self):
|
||||
return self._value
|
||||
|
||||
def __le__(self, other):
|
||||
return self.value <= other
|
||||
def set(self, new_value):
|
||||
self._value = new_value
|
||||
|
||||
def __bool__(self):
|
||||
return bool(self._value)
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.value == other
|
||||
return self._value == other
|
||||
|
||||
def __ne__(self, other):
|
||||
return self.value != other
|
||||
return self._value != other
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self._value)
|
||||
|
||||
def __lt__(self, other):
|
||||
return self._value < other
|
||||
|
||||
def __le__(self, other):
|
||||
return self._value <= other
|
||||
|
||||
def __gt__(self, other):
|
||||
return self.value > other
|
||||
return self._value > other
|
||||
|
||||
def __ge__(self, other):
|
||||
return self.value >= other
|
||||
return self._value >= other
|
||||
|
||||
def __add__(self, other):
|
||||
return self.value + other
|
||||
return self._value + other
|
||||
|
||||
def __radd__(self, other):
|
||||
return other + self.value
|
||||
return other + self._value
|
||||
|
||||
def __iadd__(self, other):
|
||||
self.value += other
|
||||
self._value += other
|
||||
return self
|
||||
|
||||
def __sub__(self, other):
|
||||
return self.value - other
|
||||
return self._value - other
|
||||
|
||||
def __rsub__(self, other):
|
||||
return other - self.value
|
||||
return other - self._value
|
||||
|
||||
def __isub__(self, other):
|
||||
self.value -= other
|
||||
self._value -= other
|
||||
return self
|
||||
|
||||
def __mul__(self, other):
|
||||
return self.value * other
|
||||
return self._value * other
|
||||
|
||||
def __rmul__(self, other):
|
||||
return other * self.value
|
||||
return other * self._value
|
||||
|
||||
def __imul__(self, other):
|
||||
self.value *= other
|
||||
self._value *= other
|
||||
return self
|
||||
|
||||
def __matmul__(self, other):
|
||||
return self.value @ other
|
||||
return self._value @ other
|
||||
|
||||
def __rmatmul__(self, other):
|
||||
return other @ self.value
|
||||
return other @ self._value
|
||||
|
||||
def __imatmul__(self, other):
|
||||
self.value @= other
|
||||
self._value @= other
|
||||
return self
|
||||
|
||||
def __truediv__(self, other):
|
||||
return self.value / other
|
||||
return self._value / other
|
||||
|
||||
def __rtruediv__(self, other):
|
||||
return other / self.value
|
||||
return other / self._value
|
||||
|
||||
def __itruediv__(self, other):
|
||||
self.value /= other
|
||||
self._value /= other
|
||||
return self
|
||||
|
||||
def __floordiv__(self, other):
|
||||
return self.value // other
|
||||
return self._value // other
|
||||
|
||||
def __rfloordiv__(self, other):
|
||||
return other // self.value
|
||||
return other // self._value
|
||||
|
||||
def __ifloordiv__(self, other):
|
||||
self.value //= other
|
||||
self._value //= other
|
||||
return self
|
||||
|
||||
def __mod__(self, other):
|
||||
return self.value % other
|
||||
return self._value % other
|
||||
|
||||
def __rmod__(self, other):
|
||||
return other % self.value
|
||||
return other % self._value
|
||||
|
||||
def __imod__(self, other):
|
||||
self.value %= other
|
||||
self._value %= other
|
||||
return self
|
||||
|
||||
def __divmod__(self, other):
|
||||
return divmod(self.value, other)
|
||||
return divmod(self._value, other)
|
||||
|
||||
def __rdivmod__(self, other):
|
||||
return divmod(other, self.value)
|
||||
return divmod(other, self._value)
|
||||
|
||||
def __pow__(self, other):
|
||||
return self.value**other
|
||||
return self._value**other
|
||||
|
||||
def __rpow__(self, other):
|
||||
return other**self.value
|
||||
return other**self._value
|
||||
|
||||
def __ipow__(self, other):
|
||||
self.value **= other
|
||||
self._value **= other
|
||||
return self
|
||||
|
||||
def __lshift__(self, other):
|
||||
return self.value << other
|
||||
return self._value << other
|
||||
|
||||
def __rlshift__(self, other):
|
||||
return other << self.value
|
||||
return other << self._value
|
||||
|
||||
def __ilshift__(self, other):
|
||||
self.value <<= other
|
||||
self._value <<= other
|
||||
return self
|
||||
|
||||
def __rshift__(self, other):
|
||||
return self.value >> other
|
||||
return self._value >> other
|
||||
|
||||
def __rrshift__(self, other):
|
||||
return other >> self.value
|
||||
return other >> self._value
|
||||
|
||||
def __irshift__(self, other):
|
||||
self.value >>= other
|
||||
self._value >>= other
|
||||
return self
|
||||
|
||||
def __and__(self, other):
|
||||
return self.value & other
|
||||
return self._value & other
|
||||
|
||||
def __rand__(self, other):
|
||||
return other & self.value
|
||||
return other & self._value
|
||||
|
||||
def __iand__(self, other):
|
||||
self.value &= other
|
||||
self._value &= other
|
||||
return self
|
||||
|
||||
def __xor__(self, other):
|
||||
return self.value ^ other
|
||||
return self._value ^ other
|
||||
|
||||
def __rxor__(self, other):
|
||||
return other ^ self.value
|
||||
return other ^ self._value
|
||||
|
||||
def __ixor__(self, other):
|
||||
self.value ^= other
|
||||
self._value ^= other
|
||||
return self
|
||||
|
||||
def __or__(self, other):
|
||||
return self.value | other
|
||||
return self._value | other
|
||||
|
||||
def __ror__(self, other):
|
||||
return other | self.value
|
||||
return other | self._value
|
||||
|
||||
def __ior__(self, other):
|
||||
self.value |= other
|
||||
self._value |= other
|
||||
return self
|
||||
|
||||
def __neg__(self):
|
||||
return -self.value
|
||||
return -self._value
|
||||
|
||||
def __pos__(self):
|
||||
return +self.value
|
||||
return +self._value
|
||||
|
||||
def __abs__(self):
|
||||
return abs(self.value)
|
||||
return abs(self._value)
|
||||
|
||||
def __invert__(self):
|
||||
return ~self.value
|
||||
return ~self._value
|
||||
|
||||
def __len__(self):
|
||||
return len(self._value)
|
||||
|
||||
|
||||
def get_attribute_or_create(obj, attr, default):
|
||||
@ -207,4 +227,3 @@ def get_attribute_or_create(obj, attr, default):
|
||||
except AttributeError:
|
||||
setattr(obj, attr, default)
|
||||
return getattr(obj, attr)
|
||||
|
||||
|
||||
8
bonobo/util/pkgs.py
Normal file
8
bonobo/util/pkgs.py
Normal file
@ -0,0 +1,8 @@
|
||||
import pkg_resources
|
||||
from packaging.utils import canonicalize_name
|
||||
|
||||
bonobo_packages = {}
|
||||
for p in pkg_resources.working_set:
|
||||
name = canonicalize_name(p.project_name)
|
||||
if name.startswith('bonobo'):
|
||||
bonobo_packages[name] = p
|
||||
22
bonobo/util/python.py
Normal file
22
bonobo/util/python.py
Normal file
@ -0,0 +1,22 @@
|
||||
import inspect
|
||||
import os
|
||||
import runpy
|
||||
|
||||
|
||||
class _RequiredModule:
|
||||
def __init__(self, dct):
|
||||
self.__dict__ = dct
|
||||
|
||||
|
||||
class _RequiredModulesRegistry(dict):
|
||||
def require(self, name):
|
||||
if name not in self:
|
||||
bits = name.split('.')
|
||||
pathname = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.stack()[1][0])))
|
||||
filename = os.path.join(pathname, *bits[:-1], bits[-1] + '.py')
|
||||
self[name] = _RequiredModule(runpy.run_path(filename, run_name=name))
|
||||
return self[name]
|
||||
|
||||
|
||||
registry = _RequiredModulesRegistry()
|
||||
require = registry.require
|
||||
31
bonobo/util/statistics.py
Normal file
31
bonobo/util/statistics.py
Normal file
@ -0,0 +1,31 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# copyright 2012-2014 romain dorgueil
|
||||
#
|
||||
# licensed under the apache license, version 2.0 (the "license");
|
||||
# you may not use this file except in compliance with the license.
|
||||
# you may obtain a copy of the license at
|
||||
#
|
||||
# http://www.apache.org/licenses/license-2.0
|
||||
#
|
||||
# unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the license is distributed on an "as is" basis,
|
||||
# without warranties or conditions of any kind, either express or implied.
|
||||
# see the license for the specific language governing permissions and
|
||||
# limitations under the license.
|
||||
|
||||
|
||||
class WithStatistics:
|
||||
def __init__(self, *names):
|
||||
self.statistics_names = names
|
||||
self.statistics = {name: 0 for name in names}
|
||||
|
||||
def get_statistics(self, *args, **kwargs):
|
||||
return ((name, self.statistics[name]) for name in self.statistics_names)
|
||||
|
||||
def get_statistics_as_string(self, *args, **kwargs):
|
||||
stats = tuple('{0}={1}'.format(name, cnt) for name, cnt in self.get_statistics(*args, **kwargs) if cnt > 0)
|
||||
return (kwargs.get('prefix', '') + ' '.join(stats)) if len(stats) else ''
|
||||
|
||||
def increment(self, name):
|
||||
self.statistics[name] += 1
|
||||
@ -1,5 +1,7 @@
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from bonobo import open_fs
|
||||
from bonobo.execution.node import NodeExecutionContext
|
||||
|
||||
|
||||
@ -7,3 +9,29 @@ class CapturingNodeExecutionContext(NodeExecutionContext):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.send = MagicMock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def optional_contextmanager(cm, *, ignore=False):
|
||||
if cm is None or ignore:
|
||||
yield
|
||||
else:
|
||||
with cm:
|
||||
yield
|
||||
|
||||
|
||||
class FilesystemTester:
|
||||
def __init__(self, extension='txt', mode='w'):
|
||||
self.extension = extension
|
||||
self.input_data = ''
|
||||
self.mode = mode
|
||||
|
||||
def get_services_for_reader(self, tmpdir):
|
||||
fs, filename = open_fs(tmpdir), 'input.' + self.extension
|
||||
with fs.open(filename, self.mode) as fp:
|
||||
fp.write(self.input_data)
|
||||
return fs, filename, {'fs': fs}
|
||||
|
||||
def get_services_for_writer(self, tmpdir):
|
||||
fs, filename = open_fs(tmpdir), 'output.' + self.extension
|
||||
return fs, filename, {'fs': fs}
|
||||
|
||||
Reference in New Issue
Block a user