|
|
|
@ -1,5 +1,5 @@
|
|
|
|
Environment Variables
|
|
|
|
Environment Variables
|
|
|
|
=======================
|
|
|
|
=====================
|
|
|
|
|
|
|
|
|
|
|
|
Best practice holds that variables should be passed to graphs via environment variables.
|
|
|
|
Best practice holds that variables should be passed to graphs via environment variables.
|
|
|
|
Doing this is important for keeping sensitive data out of the code - such as an
|
|
|
|
Doing this is important for keeping sensitive data out of the code - such as an
|
|
|
|
@ -10,13 +10,16 @@ are also the means by-which arguments can be passed to graphs.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Passing / Setting Environment Variables
|
|
|
|
Passing / Setting Environment Variables
|
|
|
|
::::::::::::::::::::::::::::::::::::::::::::
|
|
|
|
:::::::::::::::::::::::::::::::::::::::
|
|
|
|
|
|
|
|
|
|
|
|
Setting environment variables for your graphs to use can be done in a variety of ways and which one used can vary
|
|
|
|
Setting environment variables for your graphs to use can be done in a variety of ways and which one used can vary
|
|
|
|
based-upon context. Perhaps the most immediate and simple way to set/override a variable for a given graph is
|
|
|
|
based-upon context. Perhaps the most immediate and simple way to set/override a variable for a given graph is
|
|
|
|
simply to use the optional ``--env`` argument when running bonobo from the shell (bash, command prompt, etc).
|
|
|
|
simply to use the optional ``--env`` argument when running bonobo from the shell (bash, command prompt, etc).
|
|
|
|
``--env`` (or ``-e`` for short) should then be followed by the variable name and value using the
|
|
|
|
``--env`` (or ``-e`` for short) should then be followed by the variable name and value using the
|
|
|
|
syntax ``VAR_NAME=VAR_VALUE``. Multiple environment variables can be passed by using multiple ``--env`` / ``-e`` flags (i.e. ``bonobo run --env FIZZ=buzz ...`` and ``bonobo run --env FIZZ=buzz --env Foo=bar ...``). Additionally, in bash you can also set environment variables by listing those you wish to set before the `bonobo run` command with space separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``).
|
|
|
|
syntax ``VAR_NAME=VAR_VALUE``. Multiple environment variables can be passed by using multiple ``--env`` / ``-e`` flags
|
|
|
|
|
|
|
|
(i.e. ``bonobo run --env FIZZ=buzz ...`` and ``bonobo run --env FIZZ=buzz --env Foo=bar ...``). Additionally, in bash
|
|
|
|
|
|
|
|
you can also set environment variables by listing those you wish to set before the `bonobo run` command with space
|
|
|
|
|
|
|
|
separating the key-value pairs (i.e. ``FIZZ=buzz bonobo run ...`` or ``FIZZ=buzz FOO=bar bonobo run ...``).
|
|
|
|
|
|
|
|
|
|
|
|
The Examples below demonstrate setting one or multiple variables using both of these methods:
|
|
|
|
The Examples below demonstrate setting one or multiple variables using both of these methods:
|
|
|
|
|
|
|
|
|
|
|
|
@ -28,16 +31,16 @@ The Examples below demonstrate setting one or multiple variables using both of t
|
|
|
|
# Using multiple environment variables via -e (env) flag:
|
|
|
|
# Using multiple environment variables via -e (env) flag:
|
|
|
|
bonobo run csvsanitizer -e SRC_FILE=inventory.txt -e DST_FILE=inventory_processed.csv
|
|
|
|
bonobo run csvsanitizer -e SRC_FILE=inventory.txt -e DST_FILE=inventory_processed.csv
|
|
|
|
|
|
|
|
|
|
|
|
# Using one environment variable in bash (*bash only):
|
|
|
|
# Using one environment variable inline (bash only):
|
|
|
|
SECRET_TOKEN=secret123 bonobo run csvsanitizer
|
|
|
|
SECRET_TOKEN=secret123 bonobo run csvsanitizer
|
|
|
|
|
|
|
|
|
|
|
|
# Using multiple environment variables in bash (*bash only):
|
|
|
|
# Using multiple environment variables inline (bash only):
|
|
|
|
SRC_FILE=inventory.txt DST_FILE=inventory_processed.csv bonobo run csvsanitizer
|
|
|
|
SRC_FILE=inventory.txt DST_FILE=inventory_processed.csv bonobo run csvsanitizer
|
|
|
|
|
|
|
|
|
|
|
|
*Though not-yet implemented, the bonobo roadmap includes implementing environment / .env files as well.*
|
|
|
|
*Though not-yet implemented, the bonobo roadmap includes implementing environment / .env files as well.*
|
|
|
|
|
|
|
|
|
|
|
|
Accessing Environment Variables from within the Graph Context
|
|
|
|
Accessing Environment Variables from within the Graph Context
|
|
|
|
:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
|
|
:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
|
|
|
|
|
|
|
|
|
|
|
|
Environment variables, whether set globally or only for the scope of the graph,
|
|
|
|
Environment variables, whether set globally or only for the scope of the graph,
|
|
|
|
can be can be accessed using any of the normal means. It is important to note
|
|
|
|
can be can be accessed using any of the normal means. It is important to note
|
|
|
|
@ -50,26 +53,22 @@ function and used to get data from the database.
|
|
|
|
|
|
|
|
|
|
|
|
import os
|
|
|
|
import os
|
|
|
|
|
|
|
|
|
|
|
|
from bonobo import Graph, run
|
|
|
|
import bonobo
|
|
|
|
|
|
|
|
from bonobo.config import use
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def extract():
|
|
|
|
DB_USER = os.getenv('DB_USER')
|
|
|
|
database_user = os.getenv('DB_USER')
|
|
|
|
DB_PASS = os.getenv('DB_PASS')
|
|
|
|
database_password = os.getenv('DB_PASS')
|
|
|
|
|
|
|
|
# ...
|
|
|
|
|
|
|
|
# (connect to database using database_user and database_password)
|
|
|
|
|
|
|
|
# (get data from database)
|
|
|
|
|
|
|
|
# ...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return database_data
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load(database_data: dict):
|
|
|
|
@use('database')
|
|
|
|
for k, v in database_data.items():
|
|
|
|
def extract(database):
|
|
|
|
print('{key} = {value}'.format(key=k, value=v))
|
|
|
|
with database.connect(DB_USER, DB_PASS) as conn:
|
|
|
|
|
|
|
|
yield from conn.query_all()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
graph = Graph(extract, load)
|
|
|
|
graph = bonobo.Graph(
|
|
|
|
|
|
|
|
extract,
|
|
|
|
|
|
|
|
bonobo.PrettyPrinter(),
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
|
|
|
run(graph)
|
|
|
|
|