diff --git a/Projectfile b/Projectfile index c1abd79..6873522 100644 --- a/Projectfile +++ b/Projectfile @@ -34,6 +34,7 @@ python.setup( 'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register', + 'download = bonobo.commands.download:register', ], } ) diff --git a/bonobo/commands/download.py b/bonobo/commands/download.py new file mode 100644 index 0000000..fd51951 --- /dev/null +++ b/bonobo/commands/download.py @@ -0,0 +1,39 @@ +import io +import re + +import requests + +import bonobo + +EXAMPLES_BASE_URL = 'https://raw.githubusercontent.com/python-bonobo/bonobo/master/bonobo/examples/' +"""The URL to our git repository, in raw mode.""" + + +def _write_response(response, fout): + """Read the response and write it to the output stream in chunks.""" + for chunk in response.iter_content(io.DEFAULT_BUFFER_SIZE): + fout.write(chunk) + + +def _open_url(url): + """Open a HTTP connection to the URL and return a file-like object.""" + response = requests.get(url, stream=True) + if response.status_code != 200: + raise IOError('unable to download {}, HTTP {}'.format(url, response.status_code)) + return response + + +def execute(path, *args, **kwargs): + path = path.lstrip('/') + if not path.startswith('examples'): + raise ValueError('download command currently supports examples only') + examples_path = re.sub('^examples/', '', path) + output_path = bonobo.get_examples_path(examples_path) + with _open_url(EXAMPLES_BASE_URL + examples_path) as response, open(output_path, 'wb') as fout: + _write_response(response, fout) + print('saved to {}'.format(output_path)) + + +def register(parser): + parser.add_argument('path', help='The relative path of the thing to download.') + return execute diff --git a/docs/changelog.rst b/docs/changelog.rst index a049822..60a8d2b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,17 @@ Changelog ========= +Unreleased +:::::::::: + +New features +------------ + +Command line +............ + +* `bonobo download /examples/datasets/coffeeshops.txt` now downloads the coffeeshops example + v.0.5.0 - 5 october 2017 :::::::::::::::::::::::: diff --git a/docs/tutorial/tut02.rst b/docs/tutorial/tut02.rst index 9ad5ae3..7f51558 100644 --- a/docs/tutorial/tut02.rst +++ b/docs/tutorial/tut02.rst @@ -59,13 +59,7 @@ available in **Bonobo**'s repository: .. code-block:: shell-session - $ curl https://raw.githubusercontent.com/python-bonobo/bonobo/master/bonobo/examples/datasets/coffeeshops.txt > `python3 -c 'import bonobo; print(bonobo.get_examples_path("datasets/coffeeshops.txt"))'` - -.. note:: - - The "example dataset download" step will be easier in the future. - - https://github.com/python-bonobo/bonobo/issues/134 + $ bonobo download examples/datasets/coffeeshops.txt .. literalinclude:: ../../bonobo/examples/tutorials/tut02e01_read.py :language: python diff --git a/setup.py b/setup.py index 8278209..ba95cb8 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,8 @@ setup( 'bonobo.commands': [ 'convert = bonobo.commands.convert:register', 'init = bonobo.commands.init:register', 'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register', - 'version = bonobo.commands.version:register' + 'version = bonobo.commands.version:register', + 'download = bonobo.commands.download:register', ], 'console_scripts': ['bonobo = bonobo.commands:entrypoint'] }, diff --git a/tests/test_commands.py b/tests/test_commands.py index a96634c..c78fa5f 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -4,7 +4,7 @@ import os import runpy import sys from contextlib import redirect_stdout, redirect_stderr -from unittest.mock import patch +from unittest.mock import patch, Mock import pkg_resources import pytest @@ -13,6 +13,7 @@ from cookiecutter.exceptions import OutputDirExistsException from bonobo import __main__, __version__, get_examples_path from bonobo.commands import entrypoint from bonobo.commands.run import DEFAULT_GRAPH_FILENAMES +from bonobo.commands.download import EXAMPLES_BASE_URL def runner(f): @@ -152,6 +153,43 @@ def test_version(runner): assert __version__ in out +@all_runners +def test_download_works_for_examples(runner): + expected_bytes = b'hello world' + + class MockResponse(object): + def __init__(self): + self.status_code = 200 + + def iter_content(self, *args, **kwargs): + return [expected_bytes] + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + pass + + fout = io.BytesIO() + fout.close = lambda: None + + with patch('bonobo.commands.download._open_url') as mock_open_url, \ + patch('bonobo.commands.download.open') as mock_open: + mock_open_url.return_value = MockResponse() + mock_open.return_value = fout + runner('download', 'examples/datasets/coffeeshops.txt') + expected_url = EXAMPLES_BASE_URL + 'datasets/coffeeshops.txt' + mock_open_url.assert_called_once_with(expected_url) + + assert fout.getvalue() == expected_bytes + + +@all_runners +def test_download_fails_non_example(runner): + with pytest.raises(ValueError): + runner('download', '/something/entirely/different.txt') + + @all_runners class TestDefaultEnvFile(object): def test_run_file_with_default_env_file(self, runner):