Merge branch 'develop' of github.com:python-bonobo/bonobo into develop

Projectfile

@@ -45,6 +45,7 @@ python.add_requirements(
     'psutil >=5.2,<6.0',
     'requests >=2.0,<3.0',
     'stevedore >=1.21,<2.0',
+    'python-dotenv >=0.7.1,<1.0',
     dev=[
         'cookiecutter >=1.5,<1.6',
         'pytest-sugar >=0.8,<0.9',

bonobo/commands/run.py

@@ -1,7 +1,10 @@
+import codecs
 import os
+from pathlib import Path
 
 import bonobo
 from bonobo.constants import DEFAULT_SERVICES_ATTR, DEFAULT_SERVICES_FILENAME
+from dotenv import load_dotenv
 
 DEFAULT_GRAPH_FILENAMES = (
     '__main__.py',
@@ -43,8 +46,8 @@ def _install_requirements(requirements):
     importlib.reload(site)
 
 
-def read(filename, module, install=False, quiet=False, verbose=False, env=None):
+def read(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None):
-    import re
+
     import runpy
     from bonobo import Graph, settings
 
@@ -54,12 +57,6 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None):
     if verbose:
         settings.DEBUG.set(True)
 
-    if env:
-        quote_killer = re.compile('["\']')
-        for e in env:
-            var_name, var_value = e.split('=')
-            os.environ[var_name] = quote_killer.sub('', var_value)
-
     if filename:
         if os.path.isdir(filename):
             if install:
@@ -83,6 +80,22 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None):
     else:
         raise RuntimeError('UNEXPECTED: argparse should not allow this.')
 
+    env_dir = Path(filename).parent or Path(module).parent
+    if default_env_file:
+        for f in default_env_file:
+            env_file_path = str(env_dir.joinpath(f))
+            load_dotenv(env_file_path)
+    if default_env:
+        for e in default_env:
+            set_env_var(e)
+    if env_file:
+        for f in env_file:
+            env_file_path = str(env_dir.joinpath(f))
+            load_dotenv(env_file_path, override=True)
+    if env:
+        for e in env:
+            set_env_var(e, override=True)
+
     graphs = dict((k, v) for k, v in context.items() if isinstance(v, Graph))
 
     assert len(graphs) == 1, (
@@ -99,8 +112,25 @@ def read(filename, module, install=False, quiet=False, verbose=False, env=None):
     return graph, plugins, services
 
 
-def execute(filename, module, install=False, quiet=False, verbose=False, env=None):
+def set_env_var(e, override=False):
-    graph, plugins, services = read(filename, module, install, quiet, verbose, env)
+    __escape_decoder = codecs.getdecoder('unicode_escape')
+    ename, evalue = e.split('=', 1)
+
+    def decode_escaped(escaped):
+        return __escape_decoder(escaped)[0]
+
+    if len(evalue) > 0:
+        if evalue[0] == evalue[len(evalue) - 1] in ['"', "'"]:
+            evalue = decode_escaped(evalue[1:-1])
+
+    if override:
+        os.environ[ename] = evalue
+    else:
+        os.environ.setdefault(ename, evalue)
+
+
+def execute(filename, module, install=False, quiet=False, verbose=False, default_env_file=None, default_env=None, env_file=None, env=None):
+    graph, plugins, services = read(filename, module, install, quiet, verbose, default_env_file, default_env, env_file, env)
 
     return bonobo.run(graph, plugins=plugins, services=services)
 
@@ -109,6 +139,9 @@ def register_generic_run_arguments(parser, required=True):
     source_group = parser.add_mutually_exclusive_group(required=required)
     source_group.add_argument('filename', nargs='?', type=str)
     source_group.add_argument('--module', '-m', type=str)
+    parser.add_argument('--default-env-file', action='append')
+    parser.add_argument('--default-env', action='append')
+    parser.add_argument('--env-file', action='append')
     parser.add_argument('--env', '-e', action='append')
     return parser
 

bonobo/examples/env_vars/get_passed_env.py

@@ -1,20 +0,0 @@
-import os
-
-import bonobo
-
-
-def extract():
-    env_test_user = os.getenv('ENV_TEST_USER')
-    env_test_number = os.getenv('ENV_TEST_NUMBER')
-    env_test_string = os.getenv('ENV_TEST_STRING')
-    return env_test_user, env_test_number, env_test_string
-
-
-def load(s: str):
-    print(s)
-
-
-graph = bonobo.Graph(extract, load)
-
-if __name__ == '__main__':
-    bonobo.run(graph)

bonobo/examples/environment/env_files/.env_one

@@ -0,0 +1,3 @@
+MY_SECRET=321
+TEST_USER_PASSWORD=sweetpassword
+PATH=marzo

bonobo/examples/environment/env_files/.env_two

@@ -0,0 +1,2 @@
+TEST_USER_PASSWORD=not_sweet_password
+PATH='abril'

bonobo/examples/environment/env_files/get_passed_env_file.py

@@ -0,0 +1,21 @@
+import os
+
+import bonobo
+
+
+def extract():
+    my_secret = os.getenv('MY_SECRET')
+    test_user_password = os.getenv('TEST_USER_PASSWORD')
+    path = os.getenv('PATH')
+
+    return my_secret, test_user_password, path
+
+
+def load(s: str):
+    print(s)
+
+
+graph = bonobo.Graph(extract, load)
+
+if __name__ == '__main__':
+    bonobo.run(graph)

bonobo/examples/environment/env_vars/get_passed_env.py

@@ -0,0 +1,21 @@
+import os
+
+import bonobo
+
+
+def extract():
+    env_test_user = os.getenv('ENV_TEST_USER', 'user')
+    env_test_number = os.getenv('ENV_TEST_NUMBER', 'number')
+    env_test_string = os.getenv('ENV_TEST_STRING', 'string')
+    env_user = os.getenv('USER')
+    return env_test_user, env_test_number, env_test_string, env_user
+
+
+def load(s: str):
+    print(s)
+
+
+graph = bonobo.Graph(extract, load)
+
+if __name__ == '__main__':
+    bonobo.run(graph)

docs/guide/environment.rst

@@ -23,25 +23,76 @@ simply to use the optional ``--env`` argument when running bonobo from the shell
 syntax ``VAR_NAME=VAR_VALUE``. Multiple environment variables can be passed by using multiple ``--env`` / ``-e`` flags
 (i.e. ``bonobo run --env FIZZ=buzz ...`` and ``bonobo run --env FIZZ=buzz --env Foo=bar ...``). Additionally, in bash
 you can also set environment variables by listing those you wish to set before the `bonobo run` command with space
-separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``).
+separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``). Additionally,
+bonobo is able to pull environment variables from local '.env' files rather than having to pass each key-value pair
+individually at runtime. Importantly, a strict 'order of priority' is followed when setting environment variables so
+it is advisable to read and understand the order listed below to prevent
+
+
+The order of priority is from lower to higher with the higher "winning" if set:
+
+    1. default values
+            ``os.getenv("VARNAME", default_value)``
+            The user/writer/creator of the graph is responsible for setting these.
+
+    2. ``--default-env-file`` values
+            Specify file to read default env values from. Each env var in the file is used if the var isn't already a corresponding value set at the system environment (system environment vars not overwritten).
+
+    3. ``--default-env`` values
+            Works like #2 but the default ``NAME=var`` are passed individually, with one ``key=value`` pair for each ``--default-env`` flag rather than gathered from a specified file.
+
+    4. system environment values
+            Env vars already set at the system level. It is worth noting that passed env vars via ``NAME=value bonobo run ...`` falls here in the order of priority.
+
+    5. ``--env-file`` values
+            Env vars specified here are set like those in #2 albeit that these values have priority over those set at the system level.
+
+    6. ``--env`` values
+            Env vars set using the ``--env`` / ``-e`` flag work like #3 but take priority over all other env vars.
+
+
+
+Examples
+::::::::
 
 The Examples below demonstrate setting one or multiple variables using both of these methods:
 
 .. code-block:: bash
 
-    # Using one environment variable via --env flag:
+    # Using one environment variable via a --env or --defualt-env flag:
     bonobo run csvsanitizer --env SECRET_TOKEN=secret123
+    bonobo run csvsanitizer --defaul-env SECRET_TOKEN=secret123
 
-    # Using multiple environment variables via -e (env) flag:
+    # Using multiple environment variables via -e (env) and --default-env flags:
     bonobo run csvsanitizer -e SRC_FILE=inventory.txt -e DST_FILE=inventory_processed.csv
-    
+    bonobo run csvsanitizer --default-env SRC_FILE=inventory.txt --default-env DST_FILE=inventory_processed.csv
-    # Using one environment variable inline (bash only):
+
+    # Using one environment variable inline (bash-like shells only):
     SECRET_TOKEN=secret123 bonobo run csvsanitizer
 
-    # Using multiple environment variables inline (bash only):
+    # Using multiple environment variables inline (bash-like shells only):
     SRC_FILE=inventory.txt DST_FILE=inventory_processed.csv bonobo run csvsanitizer
-    
+
-*Though not-yet implemented, the bonobo roadmap includes implementing environment / .env files as well.*
+    # Using an env file for default env values:
+    bonobo run csvsanitizer --default-env-file .env
+
+    # Using an env file for env values:
+    bonobo run csvsanitizer --env-file '.env.private'
+
+
+ENV File Structure
+::::::::::::::::::
+
+The file structure for env files is incredibly simple. The only text in the file
+should be `NAME=value` pairs with one pair per line like the below.
+
+.. code-block:: text
+
+    # .env
+
+    DB_USER='bonobo'
+    DB_PASS='cicero'
+
 
 Accessing Environment Variables from within the Graph Context
 :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

requirements.txt

@@ -2,6 +2,7 @@
 appdirs==1.4.3
 certifi==2017.7.27.1
 chardet==3.0.4
+click==6.7
 colorama==0.3.9
 fs==2.0.12
 idna==2.6
@@ -9,6 +10,7 @@ packaging==16.8
 pbr==3.1.1
 psutil==5.4.0
 pyparsing==2.2.0
+python-dotenv==0.7.1
 pytz==2017.2
 requests==2.18.4
 six==1.11.0

setup.py

@@ -54,7 +54,7 @@ setup(
     include_package_data=True,
     install_requires=[
         'colorama (>= 0.3, < 1.0)', 'fs (>= 2.0, < 3.0)', 'packaging (>= 16, < 17)', 'psutil (>= 5.2, < 6.0)',
-        'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)'
+        'python-dotenv (>= 0.7.1, < 1.0)', 'requests (>= 2.0, < 3.0)', 'stevedore (>= 1.21, < 2.0)'
     ],
     extras_require={
         'dev': [

tests/config/test_configurables.py

@@ -67,7 +67,7 @@ def test_defaults():
 
     assert o.required_str == 'hello'
     assert o.default_str == 'foo'
-    assert o.integer == None
+    assert o.integer is None
 
 
 def test_str_type_factory():
@@ -78,7 +78,7 @@ def test_str_type_factory():
 
     assert o.required_str == '42'
     assert o.default_str == 'foo'
-    assert o.integer == None
+    assert o.integer is None
 
 
 def test_int_type_factory():
@@ -100,8 +100,8 @@ def test_bool_type_factory():
 
     assert o.required_str == 'yes'
     assert o.default_str == 'foo'
-    assert o.integer == None
+    assert o.integer is None
-    assert o.also_required == True
+    assert o.also_required is True
 
 
 def test_option_resolution_order():
@@ -112,7 +112,7 @@ def test_option_resolution_order():
 
     assert o.required_str == 'kaboom'
     assert o.default_str == 'foo'
-    assert o.integer == None
+    assert o.integer is None
 
 
 def test_option_positional():

tests/test_commands.py

@@ -103,27 +103,238 @@ def test_version(runner, capsys):
 
 
 @all_runners
-def test_run_with_env(runner, capsys):
+class TestDefaultEnvFile(object):
-    runner(
+    def test_run_file_with_default_env_file(self, runner, capsys):
-        'run', '--quiet',
+        runner(
-        get_examples_path('env_vars/get_passed_env.py'), '--env', 'ENV_TEST_NUMBER=123', '--env',
+            'run', '--quiet', '--default-env-file', '.env_one',
-        'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
+            get_examples_path('environment/env_files/get_passed_env_file.py')
-    )
+        )
-    out, err = capsys.readouterr()
+        out, err = capsys.readouterr()
-    out = out.split('\n')
+        out = out.split('\n')
-    assert out[0] == 'cwandrews'
+        assert out[0] == '321'
-    assert out[1] == '123'
+        assert out[1] == 'sweetpassword'
-    assert out[2] == 'my_test_string'
+        assert out[2] != 'marzo'
+
+    def test_run_file_with_multiple_default_env_files(self, runner, capsys):
+        runner(
+            'run', '--quiet', '--default-env-file', '.env_one',
+            '--default-env-file', '.env_two',
+            get_examples_path('environment/env_files/get_passed_env_file.py')
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == '321'
+        assert out[1] == 'sweetpassword'
+        assert out[2] != 'marzo'
+
+    def test_run_module_with_default_env_file(self, runner, capsys):
+        runner(
+            'run', '--quiet', '-m',
+            'bonobo.examples.environment.env_files.get_passed_env_file',
+            '--default-env-file', '.env_one'
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == '321'
+        assert out[1] == 'sweetpassword'
+        assert out[2] != 'marzo'
+
+    def test_run_module_with_multiple_default_env_files(self, runner, capsys):
+        runner(
+            'run', '--quiet', '-m',
+            'bonobo.examples.environment.env_files.get_passed_env_file',
+            '--default-env-file', '.env_one', '--default-env-file', '.env_two',
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == '321'
+        assert out[1] == 'sweetpassword'
+        assert out[2] != 'marzo'
 
 
 @all_runners
-def test_run_module_with_env(runner, capsys):
+class TestEnvFile(object):
-    runner(
+    def test_run_file_with_file(self, runner, capsys):
-        'run', '--quiet', '-m', 'bonobo.examples.env_vars.get_passed_env', '--env', 'ENV_TEST_NUMBER=123', '--env',
+        runner(
-        'ENV_TEST_USER=cwandrews', '--env', "ENV_TEST_STRING='my_test_string'"
+            'run', '--quiet',
-    )
+            get_examples_path('environment/env_files/get_passed_env_file.py'),
-    out, err = capsys.readouterr()
+            '--env-file', '.env_one',
-    out = out.split('\n')
+        )
-    assert out[0] == 'cwandrews'
+        out, err = capsys.readouterr()
-    assert out[1] == '123'
+        out = out.split('\n')
-    assert out[2] == 'my_test_string'
+        assert out[0] == '321'
+        assert out[1] == 'sweetpassword'
+        assert out[2] == 'marzo'
+
+    def test_run_file_with_multiple_files(self, runner, capsys):
+        runner(
+            'run', '--quiet',
+            get_examples_path('environment/env_files/get_passed_env_file.py'),
+            '--env-file', '.env_one', '--env-file', '.env_two',
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == '321'
+        assert out[1] == 'not_sweet_password'
+        assert out[2] == 'abril'
+
+    def test_run_module_with_file(self, runner, capsys):
+        runner(
+            'run', '--quiet', '-m',
+            'bonobo.examples.environment.env_files.get_passed_env_file',
+            '--env-file', '.env_one',
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == '321'
+        assert out[1] == 'sweetpassword'
+        assert out[2] == 'marzo'
+
+    def test_run_module_with_multiple_files(self, runner, capsys):
+        runner(
+            'run', '--quiet', '-m',
+            'bonobo.examples.environment.env_files.get_passed_env_file',
+            '--env-file', '.env_one', '--env-file', '.env_two',
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == '321'
+        assert out[1] == 'not_sweet_password'
+        assert out[2] == 'abril'
+
+
+@all_runners
+class TestEnvFileCombinations(object):
+    def test_run_file_with_default_env_file_and_env_file(self, runner, capsys):
+        runner(
+            'run', '--quiet',
+            get_examples_path('environment/env_files/get_passed_env_file.py'),
+            '--default-env-file', '.env_one', '--env-file', '.env_two',
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == '321'
+        assert out[1] == 'not_sweet_password'
+        assert out[2] == 'abril'
+
+    def test_run_file_with_default_env_file_and_env_file_and_env_vars(self, runner, capsys):
+        runner(
+            'run', '--quiet',
+            get_examples_path('environment/env_files/get_passed_env_file.py'),
+            '--default-env-file', '.env_one', '--env-file', '.env_two',
+            '--env', 'TEST_USER_PASSWORD=SWEETpassWORD', '--env',
+            'MY_SECRET=444',
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == '444'
+        assert out[1] == 'SWEETpassWORD'
+        assert out[2] == 'abril'
+
+
+@all_runners
+class TestDefaultEnvVars(object):
+    def test_run_file_with_default_env_var(self, runner, capsys):
+        runner(
+            'run', '--quiet',
+            get_examples_path('environment/env_vars/get_passed_env.py'),
+            '--default-env', 'USER=clowncity', '--env', 'USER=ted'
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == 'user'
+        assert out[1] == 'number'
+        assert out[2] == 'string'
+        assert out[3] != 'clowncity'
+
+    def test_run_file_with_default_env_vars(self, runner, capsys):
+        runner(
+            'run', '--quiet',
+            get_examples_path('environment/env_vars/get_passed_env.py'),
+            '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
+            '--default-env', "ENV_TEST_STRING='my_test_string'"
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == 'cwandrews'
+        assert out[1] == '123'
+        assert out[2] == 'my_test_string'
+
+    def test_run_module_with_default_env_var(self, runner, capsys):
+        runner(
+            'run', '--quiet', '-m',
+            'bonobo.examples.environment.env_vars.get_passed_env',
+            '--env', 'ENV_TEST_NUMBER=123',
+            '--default-env', 'ENV_TEST_STRING=string'
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == 'cwandrews'
+        assert out[1] == '123'
+        assert out[2] != 'string'
+
+    def test_run_module_with_default_env_vars(self, runner, capsys):
+        runner(
+            'run', '--quiet', '-m',
+            'bonobo.examples.environment.env_vars.get_passed_env',
+            '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
+            '--default-env', "ENV_TEST_STRING='string'"
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == 'cwandrews'
+        assert out[1] == '123'
+        assert out[2] != 'string'
+
+
+@all_runners
+class TestEnvVars(object):
+    def test_run_file_with_env_var(self, runner, capsys):
+        runner(
+            'run', '--quiet',
+            get_examples_path('environment/env_vars/get_passed_env.py'),
+            '--env', 'ENV_TEST_NUMBER=123'
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] != 'test_user'
+        assert out[1] == '123'
+        assert out[2] == 'my_test_string'
+
+    def test_run_file_with_env_vars(self, runner, capsys):
+        runner(
+            'run', '--quiet',
+            get_examples_path('environment/env_vars/get_passed_env.py'),
+            '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
+            '--env', "ENV_TEST_STRING='my_test_string'"
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == 'cwandrews'
+        assert out[1] == '123'
+        assert out[2] == 'my_test_string'
+
+    def test_run_module_with_env_var(self, runner, capsys):
+        runner(
+            'run', '--quiet', '-m',
+            'bonobo.examples.environment.env_vars.get_passed_env',
+            '--env', 'ENV_TEST_NUMBER=123'
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == 'cwandrews'
+        assert out[1] == '123'
+        assert out[2] == 'my_test_string'
+
+    def test_run_module_with_env_vars(self, runner, capsys):
+        runner(
+            'run', '--quiet', '-m',
+            'bonobo.examples.environment.env_vars.get_passed_env',
+            '--env', 'ENV_TEST_NUMBER=123', '--env', 'ENV_TEST_USER=cwandrews',
+            '--env', "ENV_TEST_STRING='my_test_string'"
+        )
+        out, err = capsys.readouterr()
+        out = out.split('\n')
+        assert out[0] == 'cwandrews'
+        assert out[1] == '123'
+        assert out[2] == 'my_test_string'

tests/test_settings.py

@@ -42,9 +42,9 @@ def test_setting():
 def test_default_settings():
     settings.clear_all()
 
-    assert settings.DEBUG.get() == False
+    assert settings.DEBUG.get() is False
-    assert settings.PROFILE.get() == False
+    assert settings.PROFILE.get() is False
-    assert settings.QUIET.get() == False
+    assert settings.QUIET.get() is False
     assert settings.LOGGING_LEVEL.get() == logging._checkLevel('INFO')
 
     with patch.dict(environ, {'DEBUG': 't'}):

tests/util/test_iterators.py

@@ -18,5 +18,5 @@ def test_force_iterator_with_generator():
         yield 'ccc'
 
     iterator = force_iterator(generator())
-    assert type(iterator) == types.GeneratorType
+    assert isinstance(iterator, types.GeneratorType)
     assert list(iterator) == ['aaa', 'bbb', 'ccc']