From 9e86abca60c1f90cc9d79de86840772b4b2c867c Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sat, 28 Oct 2017 14:08:53 +0200 Subject: [PATCH 1/5] Issue #134: add a `bonobo download url` command This enables users on different platforms to download the examples in the tutorial using the same command. --- bonobo/commands/download.py | 43 +++++++++++++++++++++++++++++++++++++ docs/changelog.rst | 11 ++++++++++ setup.py | 3 ++- tests/test_commands.py | 24 +++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 bonobo/commands/download.py diff --git a/bonobo/commands/download.py b/bonobo/commands/download.py new file mode 100644 index 0000000..897d7fc --- /dev/null +++ b/bonobo/commands/download.py @@ -0,0 +1,43 @@ +import io +import re +import urllib.request + +import bonobo + +EXAMPLES_BASE_URL = 'https://raw.githubusercontent.com/python-bonobo/bonobo/master/bonobo/examples/' +"""The URL to our git repository, in raw mode.""" + + +def _save_stream(fin, fout): + """Read the input stream and write it to the output stream block-by-block.""" + while True: + data = fin.read(io.DEFAULT_BUFFER_SIZE) + if data: + fout.write(data) + else: + break + + +def _open_url(url): + """Open a HTTP connection to the URL and return a file-like object.""" + response = urllib.request.urlopen(url) + if response.getcode() != 200: + raise IOError('unable to download {}, HTTP {}'.format(url, response.getcode())) + return response + + +def execute(path, *args, **kwargs): + path = path.lstrip('/') + if not path.startswith('examples'): + raise ValueError('download command currently supports examples only') + examples_path = re.sub('^examples/', '', path) + output_path = bonobo.get_examples_path(examples_path) + fin = _open_url(EXAMPLES_BASE_URL + examples_path) + with open(output_path, 'wb') as fout: + _save_stream(fin, fout) + print('saved to {}'.format(output_path)) + + +def register(parser): + parser.add_argument('path', help='The relative path of the thing to download.') + return execute diff --git a/docs/changelog.rst b/docs/changelog.rst index a049822..60a8d2b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,17 @@ Changelog ========= +Unreleased +:::::::::: + +New features +------------ + +Command line +............ + +* `bonobo download /examples/datasets/coffeeshops.txt` now downloads the coffeeshops example + v.0.5.0 - 5 october 2017 :::::::::::::::::::::::: diff --git a/setup.py b/setup.py index 8278209..ba95cb8 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,8 @@ setup( 'bonobo.commands': [ 'convert = bonobo.commands.convert:register', 'init = bonobo.commands.init:register', 'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register', - 'version = bonobo.commands.version:register' + 'version = bonobo.commands.version:register', + 'download = bonobo.commands.download:register', ], 'console_scripts': ['bonobo = bonobo.commands:entrypoint'] }, diff --git a/tests/test_commands.py b/tests/test_commands.py index a96634c..9b52445 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -13,6 +13,7 @@ from cookiecutter.exceptions import OutputDirExistsException from bonobo import __main__, __version__, get_examples_path from bonobo.commands import entrypoint from bonobo.commands.run import DEFAULT_GRAPH_FILENAMES +from bonobo.commands.download import EXAMPLES_BASE_URL def runner(f): @@ -152,6 +153,29 @@ def test_version(runner): assert __version__ in out +@all_runners +def test_download_works_for_examples(runner): + fout = io.BytesIO() + fout.close = lambda: None + + expected_bytes = b'hello world' + with patch('bonobo.commands.download._open_url') as mock_open_url, \ + patch('bonobo.commands.download.open') as mock_open: + mock_open_url.return_value = io.BytesIO(expected_bytes) + mock_open.return_value = fout + runner('download', 'examples/datasets/coffeeshops.txt') + expected_url = EXAMPLES_BASE_URL + 'datasets/coffeeshops.txt' + mock_open_url.assert_called_once_with(expected_url) + + assert fout.getvalue() == expected_bytes + + +@all_runners +def test_download_fails_non_example(runner): + with pytest.raises(ValueError): + runner('download', '/something/entirely/different.txt') + + @all_runners class TestDefaultEnvFile(object): def test_run_file_with_default_env_file(self, runner): From edc2321c54daf269d141f6ffd5a81960a329bff1 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sat, 28 Oct 2017 14:20:53 +0200 Subject: [PATCH 2/5] Issue #134: update documentation --- docs/tutorial/tut02.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/tutorial/tut02.rst b/docs/tutorial/tut02.rst index 9ad5ae3..7f51558 100644 --- a/docs/tutorial/tut02.rst +++ b/docs/tutorial/tut02.rst @@ -59,13 +59,7 @@ available in **Bonobo**'s repository: .. code-block:: shell-session - $ curl https://raw.githubusercontent.com/python-bonobo/bonobo/master/bonobo/examples/datasets/coffeeshops.txt > `python3 -c 'import bonobo; print(bonobo.get_examples_path("datasets/coffeeshops.txt"))'` - -.. note:: - - The "example dataset download" step will be easier in the future. - - https://github.com/python-bonobo/bonobo/issues/134 + $ bonobo download examples/datasets/coffeeshops.txt .. literalinclude:: ../../bonobo/examples/tutorials/tut02e01_read.py :language: python From 66bda718c558c79cfd296eb30b29dbd9ee2ea752 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sat, 28 Oct 2017 15:58:07 +0200 Subject: [PATCH 3/5] update Projectfile with download entry point --- Projectfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Projectfile b/Projectfile index c1abd79..6873522 100644 --- a/Projectfile +++ b/Projectfile @@ -34,6 +34,7 @@ python.setup( 'inspect = bonobo.commands.inspect:register', 'run = bonobo.commands.run:register', 'version = bonobo.commands.version:register', + 'download = bonobo.commands.download:register', ], } ) From eabc79c8ecc16d66e48a58e5e29e6501a5db312f Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sat, 28 Oct 2017 16:11:58 +0200 Subject: [PATCH 4/5] Issue #134: use requests instead of urllib --- bonobo/commands/download.py | 25 +++++++++++-------------- tests/test_commands.py | 14 +++++++++++--- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/bonobo/commands/download.py b/bonobo/commands/download.py index 897d7fc..0106645 100644 --- a/bonobo/commands/download.py +++ b/bonobo/commands/download.py @@ -1,6 +1,7 @@ import io import re -import urllib.request + +import requests import bonobo @@ -8,21 +9,17 @@ EXAMPLES_BASE_URL = 'https://raw.githubusercontent.com/python-bonobo/bonobo/mast """The URL to our git repository, in raw mode.""" -def _save_stream(fin, fout): - """Read the input stream and write it to the output stream block-by-block.""" - while True: - data = fin.read(io.DEFAULT_BUFFER_SIZE) - if data: - fout.write(data) - else: - break +def _write_response(response, fout): + """Read the response and write it to the output stream in chunks.""" + for chunk in response.iter_content(io.DEFAULT_BUFFER_SIZE): + fout.write(chunk) def _open_url(url): """Open a HTTP connection to the URL and return a file-like object.""" - response = urllib.request.urlopen(url) - if response.getcode() != 200: - raise IOError('unable to download {}, HTTP {}'.format(url, response.getcode())) + response = requests.get(url, stream=True) + if response.status_code != 200: + raise IOError('unable to download {}, HTTP {}'.format(url, response.status_code)) return response @@ -32,9 +29,9 @@ def execute(path, *args, **kwargs): raise ValueError('download command currently supports examples only') examples_path = re.sub('^examples/', '', path) output_path = bonobo.get_examples_path(examples_path) - fin = _open_url(EXAMPLES_BASE_URL + examples_path) + response = _open_url(EXAMPLES_BASE_URL + examples_path) with open(output_path, 'wb') as fout: - _save_stream(fin, fout) + _write_response(response, fout) print('saved to {}'.format(output_path)) diff --git a/tests/test_commands.py b/tests/test_commands.py index 9b52445..f68657c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -4,7 +4,7 @@ import os import runpy import sys from contextlib import redirect_stdout, redirect_stderr -from unittest.mock import patch +from unittest.mock import patch, Mock import pkg_resources import pytest @@ -155,13 +155,21 @@ def test_version(runner): @all_runners def test_download_works_for_examples(runner): + expected_bytes = b'hello world' + + class MockResponse(object): + def __init__(self): + self.status_code = 200 + + def iter_content(self, *args, **kwargs): + return [expected_bytes] + fout = io.BytesIO() fout.close = lambda: None - expected_bytes = b'hello world' with patch('bonobo.commands.download._open_url') as mock_open_url, \ patch('bonobo.commands.download.open') as mock_open: - mock_open_url.return_value = io.BytesIO(expected_bytes) + mock_open_url.return_value = MockResponse() mock_open.return_value = fout runner('download', 'examples/datasets/coffeeshops.txt') expected_url = EXAMPLES_BASE_URL + 'datasets/coffeeshops.txt' From 3e7898a9877a64fec0d96b733e77b11784b60f95 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Sat, 28 Oct 2017 16:19:05 +0200 Subject: [PATCH 5/5] Issue #134: use requests.get as a context manager --- bonobo/commands/download.py | 3 +-- tests/test_commands.py | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bonobo/commands/download.py b/bonobo/commands/download.py index 0106645..fd51951 100644 --- a/bonobo/commands/download.py +++ b/bonobo/commands/download.py @@ -29,8 +29,7 @@ def execute(path, *args, **kwargs): raise ValueError('download command currently supports examples only') examples_path = re.sub('^examples/', '', path) output_path = bonobo.get_examples_path(examples_path) - response = _open_url(EXAMPLES_BASE_URL + examples_path) - with open(output_path, 'wb') as fout: + with _open_url(EXAMPLES_BASE_URL + examples_path) as response, open(output_path, 'wb') as fout: _write_response(response, fout) print('saved to {}'.format(output_path)) diff --git a/tests/test_commands.py b/tests/test_commands.py index f68657c..c78fa5f 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -164,6 +164,12 @@ def test_download_works_for_examples(runner): def iter_content(self, *args, **kwargs): return [expected_bytes] + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + pass + fout = io.BytesIO() fout.close = lambda: None