Default service configuration in directory or file (#38).

bin/run_all_examples.sh

@@ -1,7 +1,7 @@
 #! /bin/bash
 
 __PATH__=$(cd $(dirname "$0")/..; pwd)
-EXAMPLES=$(cd $__PATH__; find bonobo/examples -name \*.py -not -name __init__.py)
+EXAMPLES=$(cd $__PATH__; find bonobo/examples -name \*.py -not -name _\*)
 
 for example in $EXAMPLES; do
   echo "===== $example ====="

bonobo/commands/run.py

@@ -1,6 +1,33 @@
 import argparse
+
+import os
+
 import bonobo
 
+DEFAULT_SERVICES_FILENAME = '_services.py'
+DEFAULT_SERVICES_ATTR = 'get_services'
+
+
+def get_default_services(filename, services=None):
+    dirname = os.path.dirname(filename)
+    services_filename = os.path.join(dirname, DEFAULT_SERVICES_FILENAME)
+    if os.path.exists(services_filename):
+        with open(services_filename) as file:
+            code = compile(file.read(), services_filename, 'exec')
+        context = {
+            '__name__': '__bonobo__',
+            '__file__': services_filename,
+        }
+        try:
+            exec(code, context)
+        except Exception as exc:
+            raise
+        return {
+            **context[DEFAULT_SERVICES_ATTR](),
+            **(services or {}),
+        }
+    return services or {}
+
 
 def execute(file, quiet=False):
     with file:
@@ -32,8 +59,7 @@ def execute(file, quiet=False):
 
     # todo if console and not quiet, then add the console plugin
     # todo when better console plugin, add it if console and just disable display
-
+    return bonobo.run(graph, plugins=[], services=get_default_services(file.name, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None))
-    return bonobo.run(graph)
 
 
 def register(parser):

bonobo/examples/datasets/_services.py

@@ -0,0 +1,9 @@
+from os.path import dirname
+
+import bonobo
+
+
+def get_services():
+    return {
+        'fs': bonobo.open_fs(dirname(__file__))
+    }

bonobo/examples/datasets/coffeeshops.py

@@ -1,4 +1,5 @@
 import bonobo
+from bonobo.commands.run import get_default_services
 from bonobo.ext.opendatasoft import OpenDataSoftAPI
 
 filename = 'coffeeshops.txt'
@@ -9,13 +10,5 @@ graph = bonobo.Graph(
     bonobo.FileWriter(path=filename),
 )
 
-
-def get_services():
-    from os.path import dirname
-    return {
-        'fs': bonobo.open_fs(dirname(__file__))
-    }
-
-
 if __name__ == '__main__':
-    bonobo.run(graph, services=get_services())
+    bonobo.run(graph, services=get_default_services(__file__))

bonobo/examples/datasets/fablabs.py

@@ -3,6 +3,7 @@ import json
 from colorama import Fore, Style
 
 import bonobo
+from bonobo.commands.run import get_default_services
 from bonobo.ext.opendatasoft import OpenDataSoftAPI
 
 try:
@@ -57,16 +58,8 @@ graph = bonobo.Graph(
     normalize,
     filter_france,
     bonobo.Tee(display),
-    bonobo.JsonWriter(path='datasets/fablabs.txt'),
+    bonobo.JsonWriter(path='fablabs.txt'),
 )
 
-
-def get_services():
-    from os.path import dirname
-    return {
-        'fs': bonobo.open_fs(dirname(__file__))
-    }
-
-
 if __name__ == '__main__':
-    bonobo.run(graph, services=get_services())
+    bonobo.run(graph, services=get_default_services(__file__))

bonobo/examples/files/_fixme_json_handlers.py

@@ -1,7 +1,7 @@
 import bonobo
+from bonobo.commands.run import get_default_services
 
-from ._services import get_services
+# XXX does not work anymore because of filesystem service, can't read HTTP
-
 url = 'https://data.toulouse-metropole.fr/explore/dataset/theatres-et-salles-de-spectacles/download?format=json&timezone=Europe/Berlin&use_labels_for_header=true'
 
 graph = bonobo.Graph(
@@ -10,4 +10,4 @@ graph = bonobo.Graph(
 )
 
 if __name__ == '__main__':
-    bonobo.run(graph)
+    bonobo.run(graph, services=get_default_services(__file__))

bonobo/examples/files/_services.py

@@ -0,0 +1,7 @@
+from bonobo import get_examples_path, open_fs
+
+
+def get_services():
+    return {
+        'fs': open_fs(get_examples_path())
+    }

bonobo/examples/files/csv_handlers.py

@@ -1,6 +1,5 @@
 import bonobo
-
+from bonobo.commands.run import get_default_services
-from ._services import get_services
 
 graph = bonobo.Graph(
     bonobo.CsvReader(path='datasets/coffeeshops.txt'),
@@ -8,4 +7,4 @@ graph = bonobo.Graph(
 )
 
 if __name__ == '__main__':
-    bonobo.run(graph, services=get_services())
+    bonobo.run(graph, services=get_default_services(__file__))

bonobo/examples/files/text.py

@@ -1,20 +0,0 @@
-from bonobo import FileReader, Graph, get_examples_path
-
-
-def skip_comments(line):
-    if not line.startswith('#'):
-        yield line
-
-
-graph = Graph(
-    FileReader(path=get_examples_path('datasets/passwd.txt')),
-    skip_comments,
-    lambda s: s.split(':'),
-    lambda l: l[0],
-    print,
-)
-
-if __name__ == '__main__':
-    import bonobo
-
-    bonobo.run(graph)

bonobo/examples/files/text_handlers.py

@@ -0,0 +1,19 @@
+import bonobo
+from bonobo.commands.run import get_default_services
+
+
+def skip_comments(line):
+    if not line.startswith('#'):
+        yield line
+
+
+graph = bonobo.Graph(
+    bonobo.FileReader(path='datasets/passwd.txt'),
+    skip_comments,
+    lambda s: s.split(':'),
+    lambda l: l[0],
+    print,
+)
+
+if __name__ == '__main__':
+    bonobo.run(graph, services=get_default_services(__file__))

bonobo/examples/tutorials/tut02_01_read.py

@@ -1,9 +1,17 @@
 import bonobo
+from bonobo.commands.run import get_default_services
 
 graph = bonobo.Graph(
-    bonobo.FileReader(path=bonobo.get_examples_path('datasets/coffeeshops.txt')),
+    bonobo.FileReader(path='datasets/coffeeshops.txt'),
     print,
 )
 
+
+def get_services():
+    return {
+        'fs': bonobo.open_fs(bonobo.get_examples_path())
+    }
+
 if __name__ == '__main__':
-    bonobo.run(graph)
+    bonobo.run(graph, services=get_default_services(__file__, get_services()))
+

docs/guide/services.rst

@@ -3,7 +3,7 @@ Services and dependencies (draft implementation)
 
 :Status: Draft implementation
 :Stability: Alpha
-:Last-Modified: 27 apr 2017
+:Last-Modified: 28 apr 2017
 
 Most probably, you'll want to use external systems within your transformations. Those systems may include databases,
 apis (using http, for example), filesystems, etc.
@@ -82,6 +82,13 @@ A dictionary, or dictionary-like, "services" named argument can be passed to the
 provided is pretty basic, and feature-less. But you can use much more evolved libraries instead of the provided
 stub, and as long as it works the same (a.k.a implements a dictionary-like interface), the system will use it.
 
+Service configuration (to be decided and implemented)
+:::::::::::::::::::::::::::::::::::::::::::::::::::::
+
+* There should be a way to configure default service implementation for a python file, a directory, a project ...
+* There should be a way to override services when running a transformation.
+* There should be a way to use environment for service configuration.
+
 Future and proposals
 ::::::::::::::::::::