diff --git a/bonobo/_api.py b/bonobo/_api.py index 9a82f41..816dfda 100644 --- a/bonobo/_api.py +++ b/bonobo/_api.py @@ -1,7 +1,7 @@ +from bonobo.execution.strategies import create_strategy from bonobo.nodes import CsvReader, CsvWriter, FileReader, FileWriter, Filter, JsonReader, JsonWriter, Limit, \ PickleReader, PickleWriter, PrettyPrinter, RateLimited, Tee, arg0_to_kwargs, count, identity, kwargs_to_arg0, noop from bonobo.nodes import LdjsonReader, LdjsonWriter -from bonobo.strategies import create_strategy from bonobo.structs import Bag, ErrorBag, Graph, Token from bonobo.util import get_name from bonobo.util.environ import parse_args, get_argument_parser @@ -35,7 +35,7 @@ def run(graph, *, plugins=None, services=None, strategy=None): You'll probably want to provide a services dictionary mapping service names to service instances. :param Graph graph: The :class:`Graph` to execute. - :param str strategy: The :class:`bonobo.strategies.base.Strategy` to use. + :param str strategy: The :class:`bonobo.execution.strategies.base.Strategy` to use. :param list plugins: The list of plugins to enhance execution. :param dict services: The implementations of services this graph will use. :return bonobo.execution.graph.GraphExecutionContext: @@ -93,10 +93,10 @@ def inspect(graph, *, format): print(_inspect_formats[format](graph)) -# bonobo.structs +# data structures register_api_group(Bag, ErrorBag, Graph, Token) -# bonobo.strategies +# execution strategies register_api(create_strategy) @@ -125,7 +125,7 @@ def open_fs(fs_url=None, *args, **kwargs): return _open_fs(expanduser(str(fs_url)), *args, **kwargs) -# bonobo.nodes +# standard transformations register_api_group( CsvReader, CsvWriter, diff --git a/bonobo/execution/contexts/__init__.py b/bonobo/execution/contexts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bonobo/execution/base.py b/bonobo/execution/contexts/base.py similarity index 99% rename from bonobo/execution/base.py rename to bonobo/execution/contexts/base.py index 74dd89b..39598bc 100644 --- a/bonobo/execution/base.py +++ b/bonobo/execution/contexts/base.py @@ -1,7 +1,6 @@ import sys from contextlib import contextmanager from logging import WARNING, ERROR -from time import sleep import mondrian from bonobo.config import create_container diff --git a/bonobo/execution/graph.py b/bonobo/execution/contexts/graph.py similarity index 96% rename from bonobo/execution/graph.py rename to bonobo/execution/contexts/graph.py index 0c5bc36..55dbf7e 100644 --- a/bonobo/execution/graph.py +++ b/bonobo/execution/contexts/graph.py @@ -1,13 +1,12 @@ from functools import partial from time import sleep -from whistle import EventDispatcher - from bonobo.config import create_container from bonobo.constants import BEGIN, END from bonobo.execution import events -from bonobo.execution.node import NodeExecutionContext -from bonobo.execution.plugin import PluginExecutionContext +from bonobo.execution.contexts.node import NodeExecutionContext +from bonobo.execution.contexts.plugin import PluginExecutionContext +from whistle import EventDispatcher class GraphExecutionContext: diff --git a/bonobo/execution/node.py b/bonobo/execution/contexts/node.py similarity index 98% rename from bonobo/execution/node.py rename to bonobo/execution/contexts/node.py index daa035f..8511825 100644 --- a/bonobo/execution/node.py +++ b/bonobo/execution/contexts/node.py @@ -1,12 +1,11 @@ import sys -import threading from queue import Empty from time import sleep from types import GeneratorType from bonobo.constants import NOT_MODIFIED, BEGIN, END from bonobo.errors import InactiveReadableError, UnrecoverableError -from bonobo.execution.base import LoopingExecutionContext +from bonobo.execution.contexts.base import LoopingExecutionContext from bonobo.structs.bags import Bag from bonobo.structs.inputs import Input from bonobo.structs.tokens import Token diff --git a/bonobo/execution/plugin.py b/bonobo/execution/contexts/plugin.py similarity index 80% rename from bonobo/execution/plugin.py rename to bonobo/execution/contexts/plugin.py index f552724..524c2e1 100644 --- a/bonobo/execution/plugin.py +++ b/bonobo/execution/contexts/plugin.py @@ -1,4 +1,4 @@ -from bonobo.execution.base import LoopingExecutionContext, recoverable +from bonobo.execution.contexts.base import LoopingExecutionContext class PluginExecutionContext(LoopingExecutionContext): diff --git a/bonobo/strategies/__init__.py b/bonobo/execution/strategies/__init__.py similarity index 76% rename from bonobo/strategies/__init__.py rename to bonobo/execution/strategies/__init__.py index 1420da6..1c5d50a 100644 --- a/bonobo/strategies/__init__.py +++ b/bonobo/execution/strategies/__init__.py @@ -1,5 +1,5 @@ -from bonobo.strategies.executor import ProcessPoolExecutorStrategy, ThreadPoolExecutorStrategy -from bonobo.strategies.naive import NaiveStrategy +from bonobo.execution.strategies.executor import ProcessPoolExecutorStrategy, ThreadPoolExecutorStrategy +from bonobo.execution.strategies.naive import NaiveStrategy __all__ = [ 'create_strategy', @@ -21,8 +21,8 @@ def create_strategy(name=None): :param name: :return: Strategy """ - from bonobo.strategies.base import Strategy import logging + from bonobo.execution.strategies.base import Strategy if isinstance(name, Strategy): return name @@ -39,4 +39,4 @@ def create_strategy(name=None): 'Invalid strategy {}. Available choices: {}.'.format(repr(name), ', '.join(sorted(STRATEGIES.keys()))) ) from exc - return factory() \ No newline at end of file + return factory() diff --git a/bonobo/strategies/base.py b/bonobo/execution/strategies/base.py similarity index 89% rename from bonobo/strategies/base.py rename to bonobo/execution/strategies/base.py index 47f7db4..0a8d2a5 100644 --- a/bonobo/strategies/base.py +++ b/bonobo/execution/strategies/base.py @@ -1,4 +1,4 @@ -from bonobo.execution.graph import GraphExecutionContext +from bonobo.execution.contexts.graph import GraphExecutionContext class Strategy: diff --git a/bonobo/strategies/executor.py b/bonobo/execution/strategies/executor.py similarity index 94% rename from bonobo/strategies/executor.py rename to bonobo/execution/strategies/executor.py index 49c5d4a..ebaba32 100644 --- a/bonobo/strategies/executor.py +++ b/bonobo/execution/strategies/executor.py @@ -1,13 +1,12 @@ import functools import logging import sys -from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor, wait, FIRST_EXCEPTION -from time import sleep +from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor -from bonobo.util import get_name -from bonobo.constants import BEGIN, END -from bonobo.strategies.base import Strategy from bonobo.structs.bags import Bag +from bonobo.constants import BEGIN, END +from bonobo.execution.strategies.base import Strategy +from bonobo.util import get_name class ExecutorStrategy(Strategy): diff --git a/bonobo/strategies/naive.py b/bonobo/execution/strategies/naive.py similarity index 91% rename from bonobo/strategies/naive.py rename to bonobo/execution/strategies/naive.py index 20477c1..bd581ff 100644 --- a/bonobo/strategies/naive.py +++ b/bonobo/execution/strategies/naive.py @@ -1,5 +1,5 @@ from bonobo.constants import BEGIN, END -from bonobo.strategies.base import Strategy +from bonobo.execution.strategies.base import Strategy from bonobo.structs.bags import Bag diff --git a/bonobo/plugins/console.py b/bonobo/plugins/console.py index dc511b7..d5f9914 100644 --- a/bonobo/plugins/console.py +++ b/bonobo/plugins/console.py @@ -170,4 +170,4 @@ class IOBuffer(): def memory_usage(): import os, psutil process = psutil.Process(os.getpid()) - return process.memory_info()[0] / float(2 ** 20) + return process.memory_info()[0] / float(2**20) diff --git a/bonobo/util/testing.py b/bonobo/util/testing.py index 7dc8f38..6dd2c8a 100644 --- a/bonobo/util/testing.py +++ b/bonobo/util/testing.py @@ -1,8 +1,8 @@ from contextlib import contextmanager from bonobo import open_fs, Token -from bonobo.execution.graph import GraphExecutionContext -from bonobo.execution.node import NodeExecutionContext +from bonobo.execution.contexts.graph import GraphExecutionContext +from bonobo.execution.contexts.node import NodeExecutionContext @contextmanager diff --git a/tests/execution/test_node.py b/tests/execution/test_node.py index fef385c..34a8ae1 100644 --- a/tests/execution/test_node.py +++ b/tests/execution/test_node.py @@ -1,5 +1,5 @@ from bonobo import Bag, Graph -from bonobo.strategies import NaiveStrategy +from bonobo.execution.strategies import NaiveStrategy from bonobo.util.testing import BufferingNodeExecutionContext, BufferingGraphExecutionContext diff --git a/tests/io/test_csv.py b/tests/io/test_csv.py index 1c4c6cc..b0b91c5 100644 --- a/tests/io/test_csv.py +++ b/tests/io/test_csv.py @@ -1,7 +1,7 @@ import pytest from bonobo import CsvReader, CsvWriter, settings -from bonobo.execution.node import NodeExecutionContext +from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext csv_tester = FilesystemTester('csv') diff --git a/tests/io/test_file.py b/tests/io/test_file.py index d7645e7..5fc2823 100644 --- a/tests/io/test_file.py +++ b/tests/io/test_file.py @@ -2,7 +2,7 @@ import pytest from bonobo import Bag, FileReader, FileWriter from bonobo.constants import BEGIN, END -from bonobo.execution.node import NodeExecutionContext +from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.util.testing import BufferingNodeExecutionContext, FilesystemTester txt_tester = FilesystemTester('txt') diff --git a/tests/io/test_json.py b/tests/io/test_json.py index b72a3de..b5b0781 100644 --- a/tests/io/test_json.py +++ b/tests/io/test_json.py @@ -2,7 +2,7 @@ import pytest from bonobo import JsonReader, JsonWriter, settings from bonobo import LdjsonReader, LdjsonWriter -from bonobo.execution.node import NodeExecutionContext +from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.util.testing import FilesystemTester, BufferingNodeExecutionContext json_tester = FilesystemTester('json') diff --git a/tests/io/test_pickle.py b/tests/io/test_pickle.py index 1f95309..8416a9f 100644 --- a/tests/io/test_pickle.py +++ b/tests/io/test_pickle.py @@ -3,7 +3,7 @@ import pickle import pytest from bonobo import Bag, PickleReader, PickleWriter -from bonobo.execution.node import NodeExecutionContext +from bonobo.execution.contexts.node import NodeExecutionContext from bonobo.util.testing import BufferingNodeExecutionContext, FilesystemTester pickle_tester = FilesystemTester('pkl', mode='wb') diff --git a/tests/plugins/test_console.py b/tests/plugins/test_console.py index 4a34f7a..543d341 100644 --- a/tests/plugins/test_console.py +++ b/tests/plugins/test_console.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock import bonobo from bonobo.execution import events -from bonobo.execution.graph import GraphExecutionContext +from bonobo.execution.contexts.graph import GraphExecutionContext from bonobo.plugins.console import ConsoleOutputPlugin from whistle import EventDispatcher diff --git a/tests/test_basicusage.py b/tests/test_basicusage.py index f002d36..7772af3 100644 --- a/tests/test_basicusage.py +++ b/tests/test_basicusage.py @@ -3,7 +3,7 @@ from unittest.mock import patch import pytest import bonobo -from bonobo.execution.graph import GraphExecutionContext +from bonobo.execution.contexts.graph import GraphExecutionContext @pytest.mark.timeout(2) diff --git a/tests/test_commands.py b/tests/test_commands.py index 64f3363..255bae7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -202,7 +202,7 @@ def env2(tmpdir): all_environ_targets = pytest.mark.parametrize( 'target', [ - (get_examples_path('environ.py'),), + (get_examples_path('environ.py'), ), ( '-m', 'bonobo.examples.environ', diff --git a/tests/test_execution.py b/tests/test_execution.py index 6fb33e4..84f40c5 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -1,7 +1,7 @@ from bonobo.config.processors import ContextProcessor from bonobo.constants import BEGIN, END -from bonobo.execution.graph import GraphExecutionContext -from bonobo.strategies import NaiveStrategy +from bonobo.execution.contexts.graph import GraphExecutionContext +from bonobo.execution.strategies import NaiveStrategy from bonobo.structs import Bag, Graph