Removing datasets from the repository.

bonobo/examples/datasets/__main__.py

@@ -1,8 +1,10 @@
+import os
+
 import bonobo
 from bonobo import examples
 from bonobo.examples.datasets.coffeeshops import get_graph as get_coffeeshops_graph
 from bonobo.examples.datasets.fablabs import get_graph as get_fablabs_graph
-from bonobo.examples.datasets.services import get_services
+from bonobo.examples.datasets.services import get_services, get_datasets_dir
 
 graph_factories = {
     'coffeeshops': get_coffeeshops_graph,
@@ -14,6 +16,7 @@ if __name__ == '__main__':
     parser.add_argument(
         '--target', '-t', choices=graph_factories.keys(), nargs='+'
     )
+    parser.add_argument('--sync', action='store_true', default=False)
 
     with bonobo.parse_args(parser) as options:
         graph_options = examples.get_graph_options(options)
@@ -22,8 +25,38 @@ if __name__ == '__main__':
             if options['target'] else sorted(graph_factories.keys())
         )
 
+        # Create a graph with all requested subgraphs
         graph = bonobo.Graph()
         for name in graph_names:
             graph = graph_factories[name](graph, **graph_options)
 
         bonobo.run(graph, services=get_services())
+
+        if options['sync']:
+            # XXX/TODO: when parallel option for node will be implemented, need to be rewriten to use a graph.
+            import boto3
+
+            s3 = boto3.client('s3')
+
+            local_dir = get_datasets_dir()
+            for root, dirs, files in os.walk(local_dir):
+                for filename in files:
+                    local_path = os.path.join(root, filename)
+                    relative_path = os.path.relpath(local_path, local_dir)
+                    s3_path = os.path.join(
+                        bonobo.__version__, relative_path
+                    )
+
+                    try:
+                        s3.head_object(
+                            Bucket='bonobo-examples', Key=s3_path
+                        )
+                    except:
+                        s3.upload_file(
+                            local_path,
+                            'bonobo-examples',
+                            s3_path,
+                            ExtraArgs={
+                                'ACL': 'public-read'
+                            }
+                        )

bonobo/examples/datasets/coffeeshops.py

@@ -58,4 +58,7 @@ if __name__ == '__main__':
     parser = examples.get_argument_parser()
 
     with bonobo.parse_args(parser) as options:
-        bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())
+        bonobo.run(
+            get_graph(**examples.get_graph_options(options)),
+            services=get_services()
+        )

bonobo/examples/datasets/fablabs.py

@@ -62,4 +62,7 @@ if __name__ == '__main__':
     parser = examples.get_argument_parser()
 
     with bonobo.parse_args(parser) as options:
-        bonobo.run(get_graph(**examples.get_graph_options(options)), services=get_services())
+        bonobo.run(
+            get_graph(**examples.get_graph_options(options)),
+            services=get_services()
+        )

bonobo/examples/datasets/services.py

@@ -1,7 +1,16 @@
+import os
+
 import bonobo
 
 
+def get_datasets_dir(*dirs):
+    home_dir = os.path.expanduser('~')
+    target_dir = os.path.join(
+        home_dir, '.cache/bonobo', bonobo.__version__, *dirs
+    )
+    os.makedirs(target_dir, exist_ok=True)
+    return target_dir
+
+
 def get_services():
-    return {
+    return {'fs': bonobo.open_fs(get_datasets_dir('datasets'))}
-        'fs': bonobo.open_fs(bonobo.get_examples_path('datasets'))
-    }