paulin/bonobo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Example datasets are now stored by bonobo minor version.

Browse Source
This commit is contained in:
Romain Dorgueil
2017-12-02 16:19:27 +01:00
parent 7a25774b0f
commit 47f3897847
2 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bonobo/examples/datasets/__main__.py
View File

@ -4,7 +4,7 @@ import bonobo
from bonobo import examples
from bonobo.examples.datasets.coffeeshops import get_graph as get_coffeeshops_graph
from bonobo.examples.datasets.fablabs import get_graph as get_fablabs_graph
from bonobo.examples.datasets.services import get_services, get_datasets_dir
from bonobo.examples.datasets.services import get_services, get_datasets_dir, get_minor_version
graph_factories = {
'coffeeshops': get_coffeeshops_graph,
@ -33,7 +33,7 @@ if __name__ == '__main__':
bonobo.run(graph, services=get_services())
if options['sync']:
# XXX/TODO: when parallel option for node will be implemented, need to be rewriten to use a graph.
# TODO: when parallel option for node will be implemented, need to be rewriten to use a graph.
import boto3
s3 = boto3.client('s3')
@ -44,7 +44,7 @@ if __name__ == '__main__':
local_path = os.path.join(root, filename)
relative_path = os.path.relpath(local_path, local_dir)
s3_path = os.path.join(
bonobo.__version__, relative_path
get_minor_version(), relative_path
)
try:

6
bonobo/examples/datasets/services.py
View File

@ -3,10 +3,14 @@ import os
import bonobo
def get_minor_version():
return '.'.join(bonobo.__version__.split('.')[:2])
def get_datasets_dir(*dirs):
home_dir = os.path.expanduser('~')
target_dir = os.path.join(
home_dir, '.cache/bonobo', bonobo.__version__, *dirs
home_dir, '.cache/bonobo', get_minor_version(), *dirs
)
os.makedirs(target_dir, exist_ok=True)
return target_dir