From d13b8b28e572e6f4ac175151de44dd6c9eba6f63 Mon Sep 17 00:00:00 2001 From: Alex Vykaliuk Date: Sat, 15 Jul 2017 12:52:58 +0200 Subject: [PATCH 1/4] Add ability to install requirements with for a requirements.txt residing in the same dir --- bonobo/commands/run.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py index 6de6bf6..fb93e77 100644 --- a/bonobo/commands/run.py +++ b/bonobo/commands/run.py @@ -26,6 +26,20 @@ def get_default_services(filename, services=None): return services or {} +def _install_requirements(requirements): + """Install requirements given a path to requirements.txt file.""" + import importlib + import pip + + pip.main(['install', '-r', requirements]) + # Some shenanigans to be sure everything is importable after this, especially .egg-link files which + # are referenced in *.pth files and apparently loaded by site.py at some magic bootstrap moment of the + # python interpreter. + pip.utils.pkg_resources = importlib.reload(pip.utils.pkg_resources) + import site + importlib.reload(site) + + def execute(filename, module, install=False, quiet=False, verbose=False): import runpy from bonobo import Graph, run, settings @@ -39,16 +53,8 @@ def execute(filename, module, install=False, quiet=False, verbose=False): if filename: if os.path.isdir(filename): if install: - import importlib - import pip requirements = os.path.join(filename, 'requirements.txt') - pip.main(['install', '-r', requirements]) - # Some shenanigans to be sure everything is importable after this, especially .egg-link files which - # are referenced in *.pth files and apparently loaded by site.py at some magic bootstrap moment of the - # python interpreter. - pip.utils.pkg_resources = importlib.reload(pip.utils.pkg_resources) - import site - importlib.reload(site) + _install_requirements(requirements) pathname = filename for filename in DEFAULT_GRAPH_FILENAMES: @@ -58,7 +64,8 @@ def execute(filename, module, install=False, quiet=False, verbose=False): if not os.path.exists(filename): raise IOError('Could not find entrypoint (candidates: {}).'.format(', '.join(DEFAULT_GRAPH_FILENAMES))) elif install: - raise RuntimeError('Cannot --install on a file (only available for dirs containing requirements.txt).') + requirements = os.path.join(os.path.dirname(filename), 'requirements.txt') + _install_requirements(requirements) context = runpy.run_path(filename, run_name='__bonobo__') elif module: context = runpy.run_module(module, run_name='__bonobo__') From 7aee728b8dd14aeed9f38866e529744017d9440e Mon Sep 17 00:00:00 2001 From: Alex Vykaliuk Date: Sat, 15 Jul 2017 14:24:44 +0200 Subject: [PATCH 2/4] Add tests for --install of run command --- tests/test_commands.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index 280308d..df59115 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,4 @@ +import os import runpy import sys from unittest.mock import patch @@ -70,6 +71,24 @@ def test_run_path(runner, capsys): assert out[2].startswith('Baz ') +@all_runners +def test_install_requirements_for_dir(runner): + dirname = get_examples_path('types') + with patch('pip.main') as pip_mock: + runner('run', '--install', dirname) + pip_mock.assert_called_once_with( + ['install', '-r', os.path.join(dirname, 'requirements.txt')]) + + +@all_runners +def test_install_requirements_for_file(runner): + dirname = get_examples_path('types') + with patch('pip.main') as pip_mock: + runner('run', '--install', os.path.join(dirname, 'strings.py')) + pip_mock.assert_called_once_with( + ['install', '-r', os.path.join(dirname, 'requirements.txt')]) + + @all_runners def test_version(runner, capsys): runner('version') From a8ed0e432220ae4e8b35653f23a69ba816da5180 Mon Sep 17 00:00:00 2001 From: Alex Vykaliuk Date: Sat, 15 Jul 2017 14:52:22 +0200 Subject: [PATCH 3/4] Move patch one level up because importlib brakes all the CI tools. --- tests/test_commands.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index df59115..e2d7b7b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -74,19 +74,19 @@ def test_run_path(runner, capsys): @all_runners def test_install_requirements_for_dir(runner): dirname = get_examples_path('types') - with patch('pip.main') as pip_mock: + with patch('bonobo.commands.run._install_requirements') as install_mock: runner('run', '--install', dirname) - pip_mock.assert_called_once_with( - ['install', '-r', os.path.join(dirname, 'requirements.txt')]) + install_mock.assert_called_once_with( + os.path.join(dirname, 'requirements.txt')) @all_runners def test_install_requirements_for_file(runner): dirname = get_examples_path('types') - with patch('pip.main') as pip_mock: + with patch('bonobo.commands.run._install_requirements') as install_mock: runner('run', '--install', os.path.join(dirname, 'strings.py')) - pip_mock.assert_called_once_with( - ['install', '-r', os.path.join(dirname, 'requirements.txt')]) + install_mock.assert_called_once_with( + os.path.join(dirname, 'requirements.txt')) @all_runners From 575462ca4cf9f5345939026ce5571bdc7e8277ad Mon Sep 17 00:00:00 2001 From: Vitalii Vokhmin Date: Sat, 15 Jul 2017 15:35:01 +0200 Subject: [PATCH 4/4] Check if PluginExecutionContext was started before shutting it down. If a `PluginExecutionContext().shutdown()` is called _before_ `PluginExecutionContext().start()` was called, this leads to an `AttributeError` exception since finalizer tries to access to attributes which were never defined. --- bonobo/execution/plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bonobo/execution/plugin.py b/bonobo/execution/plugin.py index d928f4a..a207f23 100644 --- a/bonobo/execution/plugin.py +++ b/bonobo/execution/plugin.py @@ -16,8 +16,9 @@ class PluginExecutionContext(LoopingExecutionContext): self.wrapped.initialize() def shutdown(self): - with recoverable(self.handle_error): - self.wrapped.finalize() + if self.started: + with recoverable(self.handle_error): + self.wrapped.finalize() self.alive = False def step(self):