From 357683bd02f3205208f095f18581b1543eb8ba08 Mon Sep 17 00:00:00 2001
From: Romain Dorgueil <romain@dorgueil.net>
Date: Fri, 28 Apr 2017 07:37:15 +0200
Subject: [PATCH] Default service configuration in directory or file (#38).

---
 bin/run_all_examples.sh                       |  2 +-
 bonobo/commands/run.py                        | 30 +++++++++++++++++--
 bonobo/examples/datasets/_services.py         |  9 ++++++
 bonobo/examples/datasets/coffeeshops.py       | 11 ++-----
 bonobo/examples/datasets/fablabs.py           | 13 ++------
 .../{json.py => _fixme_json_handlers.py}      |  6 ++--
 bonobo/examples/files/_services.py            |  7 +++++
 .../files/{csv.py => csv_handlers.py}         |  5 ++--
 bonobo/examples/files/text.py                 | 20 -------------
 bonobo/examples/files/text_handlers.py        | 19 ++++++++++++
 bonobo/examples/tutorials/tut02_01_read.py    | 12 ++++++--
 docs/guide/services.rst                       |  9 +++++-
 12 files changed, 92 insertions(+), 51 deletions(-)
 create mode 100644 bonobo/examples/datasets/_services.py
 rename bonobo/examples/files/{json.py => _fixme_json_handlers.py} (58%)
 create mode 100644 bonobo/examples/files/_services.py
 rename bonobo/examples/files/{csv.py => csv_handlers.py} (53%)
 delete mode 100644 bonobo/examples/files/text.py
 create mode 100644 bonobo/examples/files/text_handlers.py

diff --git a/bin/run_all_examples.sh b/bin/run_all_examples.sh
index a2c061c..ca6e549 100755
--- a/bin/run_all_examples.sh
+++ b/bin/run_all_examples.sh
@@ -1,7 +1,7 @@
 #! /bin/bash
 
 __PATH__=$(cd $(dirname "$0")/..; pwd)
-EXAMPLES=$(cd $__PATH__; find bonobo/examples -name \*.py -not -name __init__.py)
+EXAMPLES=$(cd $__PATH__; find bonobo/examples -name \*.py -not -name _\*)
 
 for example in $EXAMPLES; do
   echo "===== $example ====="
diff --git a/bonobo/commands/run.py b/bonobo/commands/run.py
index c52ab64..aa232d4 100644
--- a/bonobo/commands/run.py
+++ b/bonobo/commands/run.py
@@ -1,6 +1,33 @@
 import argparse
+
+import os
+
 import bonobo
 
+DEFAULT_SERVICES_FILENAME = '_services.py'
+DEFAULT_SERVICES_ATTR = 'get_services'
+
+
+def get_default_services(filename, services=None):
+    dirname = os.path.dirname(filename)
+    services_filename = os.path.join(dirname, DEFAULT_SERVICES_FILENAME)
+    if os.path.exists(services_filename):
+        with open(services_filename) as file:
+            code = compile(file.read(), services_filename, 'exec')
+        context = {
+            '__name__': '__bonobo__',
+            '__file__': services_filename,
+        }
+        try:
+            exec(code, context)
+        except Exception as exc:
+            raise
+        return {
+            **context[DEFAULT_SERVICES_ATTR](),
+            **(services or {}),
+        }
+    return services or {}
+
 
 def execute(file, quiet=False):
     with file:
@@ -32,8 +59,7 @@ def execute(file, quiet=False):
 
     # todo if console and not quiet, then add the console plugin
     # todo when better console plugin, add it if console and just disable display
-
-    return bonobo.run(graph)
+    return bonobo.run(graph, plugins=[], services=get_default_services(file.name, context.get(DEFAULT_SERVICES_ATTR)() if DEFAULT_SERVICES_ATTR in context else None))
 
 
 def register(parser):
diff --git a/bonobo/examples/datasets/_services.py b/bonobo/examples/datasets/_services.py
new file mode 100644
index 0000000..9998961
--- /dev/null
+++ b/bonobo/examples/datasets/_services.py
@@ -0,0 +1,9 @@
+from os.path import dirname
+
+import bonobo
+
+
+def get_services():
+    return {
+        'fs': bonobo.open_fs(dirname(__file__))
+    }
diff --git a/bonobo/examples/datasets/coffeeshops.py b/bonobo/examples/datasets/coffeeshops.py
index 294164f..9c6e187 100644
--- a/bonobo/examples/datasets/coffeeshops.py
+++ b/bonobo/examples/datasets/coffeeshops.py
@@ -1,4 +1,5 @@
 import bonobo
+from bonobo.commands.run import get_default_services
 from bonobo.ext.opendatasoft import OpenDataSoftAPI
 
 filename = 'coffeeshops.txt'
@@ -9,13 +10,5 @@ graph = bonobo.Graph(
     bonobo.FileWriter(path=filename),
 )
 
-
-def get_services():
-    from os.path import dirname
-    return {
-        'fs': bonobo.open_fs(dirname(__file__))
-    }
-
-
 if __name__ == '__main__':
-    bonobo.run(graph, services=get_services())
+    bonobo.run(graph, services=get_default_services(__file__))
diff --git a/bonobo/examples/datasets/fablabs.py b/bonobo/examples/datasets/fablabs.py
index 7f1f1a5..1ed52d7 100644
--- a/bonobo/examples/datasets/fablabs.py
+++ b/bonobo/examples/datasets/fablabs.py
@@ -3,6 +3,7 @@ import json
 from colorama import Fore, Style
 
 import bonobo
+from bonobo.commands.run import get_default_services
 from bonobo.ext.opendatasoft import OpenDataSoftAPI
 
 try:
@@ -57,16 +58,8 @@ graph = bonobo.Graph(
     normalize,
     filter_france,
     bonobo.Tee(display),
-    bonobo.JsonWriter(path='datasets/fablabs.txt'),
+    bonobo.JsonWriter(path='fablabs.txt'),
 )
 
-
-def get_services():
-    from os.path import dirname
-    return {
-        'fs': bonobo.open_fs(dirname(__file__))
-    }
-
-
 if __name__ == '__main__':
-    bonobo.run(graph, services=get_services())
+    bonobo.run(graph, services=get_default_services(__file__))
diff --git a/bonobo/examples/files/json.py b/bonobo/examples/files/_fixme_json_handlers.py
similarity index 58%
rename from bonobo/examples/files/json.py
rename to bonobo/examples/files/_fixme_json_handlers.py
index 3f6fdd9..37300f4 100644
--- a/bonobo/examples/files/json.py
+++ b/bonobo/examples/files/_fixme_json_handlers.py
@@ -1,7 +1,7 @@
 import bonobo
+from bonobo.commands.run import get_default_services
 
-from ._services import get_services
-
+# XXX does not work anymore because of filesystem service, can't read HTTP
 url = 'https://data.toulouse-metropole.fr/explore/dataset/theatres-et-salles-de-spectacles/download?format=json&timezone=Europe/Berlin&use_labels_for_header=true'
 
 graph = bonobo.Graph(
@@ -10,4 +10,4 @@ graph = bonobo.Graph(
 )
 
 if __name__ == '__main__':
-    bonobo.run(graph)
+    bonobo.run(graph, services=get_default_services(__file__))
diff --git a/bonobo/examples/files/_services.py b/bonobo/examples/files/_services.py
new file mode 100644
index 0000000..27eb71a
--- /dev/null
+++ b/bonobo/examples/files/_services.py
@@ -0,0 +1,7 @@
+from bonobo import get_examples_path, open_fs
+
+
+def get_services():
+    return {
+        'fs': open_fs(get_examples_path())
+    }
diff --git a/bonobo/examples/files/csv.py b/bonobo/examples/files/csv_handlers.py
similarity index 53%
rename from bonobo/examples/files/csv.py
rename to bonobo/examples/files/csv_handlers.py
index a4358d3..b4cef22 100644
--- a/bonobo/examples/files/csv.py
+++ b/bonobo/examples/files/csv_handlers.py
@@ -1,6 +1,5 @@
 import bonobo
-
-from ._services import get_services
+from bonobo.commands.run import get_default_services
 
 graph = bonobo.Graph(
     bonobo.CsvReader(path='datasets/coffeeshops.txt'),
@@ -8,4 +7,4 @@ graph = bonobo.Graph(
 )
 
 if __name__ == '__main__':
-    bonobo.run(graph, services=get_services())
+    bonobo.run(graph, services=get_default_services(__file__))
diff --git a/bonobo/examples/files/text.py b/bonobo/examples/files/text.py
deleted file mode 100644
index 62e5aba..0000000
--- a/bonobo/examples/files/text.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from bonobo import FileReader, Graph, get_examples_path
-
-
-def skip_comments(line):
-    if not line.startswith('#'):
-        yield line
-
-
-graph = Graph(
-    FileReader(path=get_examples_path('datasets/passwd.txt')),
-    skip_comments,
-    lambda s: s.split(':'),
-    lambda l: l[0],
-    print,
-)
-
-if __name__ == '__main__':
-    import bonobo
-
-    bonobo.run(graph)
diff --git a/bonobo/examples/files/text_handlers.py b/bonobo/examples/files/text_handlers.py
new file mode 100644
index 0000000..04b675e
--- /dev/null
+++ b/bonobo/examples/files/text_handlers.py
@@ -0,0 +1,19 @@
+import bonobo
+from bonobo.commands.run import get_default_services
+
+
+def skip_comments(line):
+    if not line.startswith('#'):
+        yield line
+
+
+graph = bonobo.Graph(
+    bonobo.FileReader(path='datasets/passwd.txt'),
+    skip_comments,
+    lambda s: s.split(':'),
+    lambda l: l[0],
+    print,
+)
+
+if __name__ == '__main__':
+    bonobo.run(graph, services=get_default_services(__file__))
diff --git a/bonobo/examples/tutorials/tut02_01_read.py b/bonobo/examples/tutorials/tut02_01_read.py
index 08f28cc..1c11a32 100644
--- a/bonobo/examples/tutorials/tut02_01_read.py
+++ b/bonobo/examples/tutorials/tut02_01_read.py
@@ -1,9 +1,17 @@
 import bonobo
+from bonobo.commands.run import get_default_services
 
 graph = bonobo.Graph(
-    bonobo.FileReader(path=bonobo.get_examples_path('datasets/coffeeshops.txt')),
+    bonobo.FileReader(path='datasets/coffeeshops.txt'),
     print,
 )
 
+
+def get_services():
+    return {
+        'fs': bonobo.open_fs(bonobo.get_examples_path())
+    }
+
 if __name__ == '__main__':
-    bonobo.run(graph)
+    bonobo.run(graph, services=get_default_services(__file__, get_services()))
+
diff --git a/docs/guide/services.rst b/docs/guide/services.rst
index 6538488..66d2671 100644
--- a/docs/guide/services.rst
+++ b/docs/guide/services.rst
@@ -3,7 +3,7 @@ Services and dependencies (draft implementation)
 
 :Status: Draft implementation
 :Stability: Alpha
-:Last-Modified: 27 apr 2017
+:Last-Modified: 28 apr 2017
 
 Most probably, you'll want to use external systems within your transformations. Those systems may include databases,
 apis (using http, for example), filesystems, etc.
@@ -82,6 +82,13 @@ A dictionary, or dictionary-like, "services" named argument can be passed to the
 provided is pretty basic, and feature-less. But you can use much more evolved libraries instead of the provided
 stub, and as long as it works the same (a.k.a implements a dictionary-like interface), the system will use it.
 
+Service configuration (to be decided and implemented)
+:::::::::::::::::::::::::::::::::::::::::::::::::::::
+
+* There should be a way to configure default service implementation for a python file, a directory, a project ...
+* There should be a way to override services when running a transformation.
+* There should be a way to use environment for service configuration.
+
 Future and proposals
 ::::::::::::::::::::