Commit 734ba518 authored by Ottomata's avatar Ottomata
Browse files

Update call.py arg parsing

Regen docs
parent 30e870e9
......@@ -23,7 +23,8 @@ To build sphinx html docs using current python env:
```
pip install .[docs] # install docs generator dependencies
cd docs
SPHINXBUILD='sphinx-build' make html
sphinx-apidoc -f -o ./source ../workflow_utils && \
SPHINXBUILD='sphinx-build' make clean html
```
To update frozen-requirements.txt, first make sure your current
......
......@@ -3,7 +3,7 @@
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXOPTS ?= -E
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source
BUILDDIR = build
......
......@@ -37,6 +37,8 @@
<li><a href="workflow_utils/artifact/locator.html">workflow_utils.artifact.locator</a></li>
<li><a href="workflow_utils/artifact/maven.html">workflow_utils.artifact.maven</a></li>
<li><a href="workflow_utils/artifact/source.html">workflow_utils.artifact.source</a></li>
<li><a href="workflow_utils/call.html">workflow_utils.call</a></li>
<li><a href="workflow_utils/conda.html">workflow_utils.conda</a></li>
<li><a href="workflow_utils/util.html">workflow_utils.util</a></li>
</ul>
......
......@@ -12,6 +12,22 @@ Subpackages
Submodules
----------
workflow\_utils.call module
---------------------------
.. automodule:: workflow_utils.call
:members:
:undoc-members:
:show-inheritance:
workflow\_utils.conda module
----------------------------
.. automodule:: workflow_utils.conda
:members:
:undoc-members:
:show-inheritance:
workflow\_utils.util module
---------------------------
......
......@@ -39,6 +39,7 @@
| <a href="#D"><strong>D</strong></a>
| <a href="#E"><strong>E</strong></a>
| <a href="#F"><strong>F</strong></a>
| <a href="#G"><strong>G</strong></a>
| <a href="#I"><strong>I</strong></a>
| <a href="#L"><strong>L</strong></a>
| <a href="#M"><strong>M</strong></a>
......@@ -76,13 +77,33 @@
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.cache.FsMavenArtifactCache.cache_key">(workflow_utils.artifact.cache.FsMavenArtifactCache class method)</a>
</li>
</ul></li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.artifact.Artifact.cache_put">cache_put() (workflow_utils.artifact.artifact.Artifact method)</a>
</li>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.artifact.Artifact.cached_url">cached_url() (workflow_utils.artifact.artifact.Artifact method)</a>
</li>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.artifact.Artifact.cached_urls">cached_urls() (workflow_utils.artifact.artifact.Artifact method)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.call.call">call() (in module workflow_utils.call)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.conda_cli">conda_cli() (in module workflow_utils.conda)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.html#workflow_utils.conda.conda_create">conda_create() (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.conda_create_and_pack_dist_env">conda_create_and_pack_dist_env() (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.conda_create_dist_env">conda_create_dist_env() (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.conda_dist">conda_dist() (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.conda_dist_env_defaults">conda_dist_env_defaults (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.conda_env_update">conda_env_update() (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.conda_install">conda_install() (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.conda_pack_defaults">conda_pack_defaults (in module workflow_utils.conda)</a>
</li>
</ul></td>
</tr></table>
......@@ -126,10 +147,14 @@
</li>
<li><a href="workflow_utils.html#workflow_utils.util.filter_files_exist">filter_files_exist() (in module workflow_utils.util)</a>
</li>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.cache.FsArtifactCache.fs">fs() (workflow_utils.artifact.cache.FsArtifactCache method)</a>
<li><a href="workflow_utils.html#workflow_utils.conda.find_conda_exe">find_conda_exe() (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.find_pip_exe">find_pip_exe() (in module workflow_utils.conda)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.cache.FsArtifactCache.fs">fs() (workflow_utils.artifact.cache.FsArtifactCache method)</a>
</li>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.cache.FsArtifactCache">FsArtifactCache (class in workflow_utils.artifact.cache)</a>
</li>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.source.FsArtifactSource">FsArtifactSource (class in workflow_utils.artifact.source)</a>
......@@ -139,13 +164,25 @@
</ul></td>
</tr></table>
<h2 id="G">G</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.html#workflow_utils.call.get_possible_sys_paths">get_possible_sys_paths() (in module workflow_utils.call)</a>
</li>
</ul></td>
</tr></table>
<h2 id="I">I</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.html#workflow_utils.util.instantiate">instantiate() (in module workflow_utils.util)</a>
<li><a href="workflow_utils.html#workflow_utils.call.import_it">import_it() (in module workflow_utils.call)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.call.include_python_env_sys_path">include_python_env_sys_path() (in module workflow_utils.call)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.html#workflow_utils.util.instantiate">instantiate() (in module workflow_utils.util)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.util.instantiate_all">instantiate_all() (in module workflow_utils.util)</a>
</li>
</ul></td>
......@@ -169,7 +206,11 @@
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.cli.main">main() (in module workflow_utils.artifact.cli)</a>
<ul>
<li><a href="workflow_utils.html#workflow_utils.call.main">(in module workflow_utils.call)</a>
</li>
</ul></li>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.maven.maven_artifact_dir">maven_artifact_dir() (in module workflow_utils.artifact.maven)</a>
</li>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.maven.maven_artifact_filename">maven_artifact_filename() (in module workflow_utils.artifact.maven)</a>
......@@ -197,6 +238,10 @@
<li><a href="workflow_utils.artifact.html#module-workflow_utils.artifact.maven">workflow_utils.artifact.maven</a>
</li>
<li><a href="workflow_utils.artifact.html#module-workflow_utils.artifact.source">workflow_utils.artifact.source</a>
</li>
<li><a href="workflow_utils.html#module-workflow_utils.call">workflow_utils.call</a>
</li>
<li><a href="workflow_utils.html#module-workflow_utils.conda">workflow_utils.conda</a>
</li>
<li><a href="workflow_utils.html#module-workflow_utils.util">workflow_utils.util</a>
</li>
......@@ -223,16 +268,24 @@
<h2 id="P">P</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.html#workflow_utils.conda.pack">pack() (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.maven.parse_maven_coordinate">parse_maven_coordinate() (in module workflow_utils.artifact.maven)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.conda.pip_cli">pip_cli() (in module workflow_utils.conda)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.html#workflow_utils.conda.pip_install">pip_install() (in module workflow_utils.conda)</a>
</li>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.cache.ArtifactCache.put">put() (workflow_utils.artifact.cache.ArtifactCache method)</a>
<ul>
<li><a href="workflow_utils.artifact.html#workflow_utils.artifact.cache.FsArtifactCache.put">(workflow_utils.artifact.cache.FsArtifactCache method)</a>
</li>
</ul></li>
<li><a href="workflow_utils.html#workflow_utils.call.python_env_sys_path">python_env_sys_path() (in module workflow_utils.call)</a>
</li>
</ul></td>
</tr></table>
......@@ -244,6 +297,8 @@
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="workflow_utils.html#workflow_utils.util.setup_logging">setup_logging() (in module workflow_utils.util)</a>
</li>
<li><a href="workflow_utils.html#workflow_utils.call.sys_path_prepend">sys_path_prepend() (in module workflow_utils.call)</a>
</li>
</ul></td>
</tr></table>
......@@ -308,8 +363,6 @@
<li><a href="workflow_utils.artifact.html#module-workflow_utils.artifact.cli">module</a>
</li>
</ul></li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li>
workflow_utils.artifact.locator
......@@ -317,6 +370,8 @@
<li><a href="workflow_utils.artifact.html#module-workflow_utils.artifact.locator">module</a>
</li>
</ul></li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li>
workflow_utils.artifact.maven
......@@ -329,6 +384,20 @@
<ul>
<li><a href="workflow_utils.artifact.html#module-workflow_utils.artifact.source">module</a>
</li>
</ul></li>
<li>
workflow_utils.call
<ul>
<li><a href="workflow_utils.html#module-workflow_utils.call">module</a>
</li>
</ul></li>
<li>
workflow_utils.conda
<ul>
<li><a href="workflow_utils.html#module-workflow_utils.conda">module</a>
</li>
</ul></li>
<li>
......
......@@ -53,6 +53,8 @@
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="workflow_utils.html#submodules">Submodules</a></li>
<li class="toctree-l2"><a class="reference internal" href="workflow_utils.html#module-workflow_utils.call">workflow_utils.call module</a></li>
<li class="toctree-l2"><a class="reference internal" href="workflow_utils.html#module-workflow_utils.conda">workflow_utils.conda module</a></li>
<li class="toctree-l2"><a class="reference internal" href="workflow_utils.html#module-workflow_utils.util">workflow_utils.util module</a></li>
<li class="toctree-l2"><a class="reference internal" href="workflow_utils.html#module-workflow_utils">Module contents</a></li>
</ul>
......
......@@ -85,6 +85,16 @@
<td>&#160;&#160;&#160;
<a href="workflow_utils.artifact.html#module-workflow_utils.artifact.source"><code class="xref">workflow_utils.artifact.source</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
<td>&#160;&#160;&#160;
<a href="workflow_utils.html#module-workflow_utils.call"><code class="xref">workflow_utils.call</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
<td>&#160;&#160;&#160;
<a href="workflow_utils.html#module-workflow_utils.conda"><code class="xref">workflow_utils.conda</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
<td>&#160;&#160;&#160;
......
......@@ -53,7 +53,8 @@ Tox is used for CI testing.</p>
<p>To build sphinx html docs using current python env:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">pip</span> <span class="n">install</span> <span class="o">.</span><span class="p">[</span><span class="n">docs</span><span class="p">]</span> <span class="c1"># install docs generator dependencies</span>
<span class="n">cd</span> <span class="n">docs</span>
<span class="n">SPHINXBUILD</span><span class="o">=</span><span class="s1">&#39;sphinx-build&#39;</span> <span class="n">make</span> <span class="n">html</span>
<span class="n">sphinx</span><span class="o">-</span><span class="n">apidoc</span> <span class="o">-</span><span class="n">f</span> <span class="o">-</span><span class="n">o</span> <span class="o">./</span><span class="n">source</span> <span class="o">../</span><span class="n">workflow_utils</span> <span class="o">&amp;&amp;</span> \
<span class="n">SPHINXBUILD</span><span class="o">=</span><span class="s1">&#39;sphinx-build&#39;</span> <span class="n">make</span> <span class="n">clean</span> <span class="n">html</span>
</pre></div>
</div>
<p>To update frozen-requirements.txt, first make sure your current
......
This diff is collapsed.
......@@ -351,7 +351,7 @@ a Maven compatible directory hierarchy.</p>
<span id="workflow-utils-artifact-cli-module"></span><h2>workflow_utils.artifact.cli module<a class="headerlink" href="#module-workflow_utils.artifact.cli" title="Permalink to this headline"></a></h2>
<dl class="py function">
<dt class="sig sig-object py" id="workflow_utils.artifact.cli.main">
<span class="sig-prename descclassname"><span class="pre">workflow_utils.artifact.cli.</span></span><span class="sig-name descname"><span class="pre">main</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">argv</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">['-M',</span> <span class="pre">'html',</span> <span class="pre">'source',</span> <span class="pre">'build']</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/workflow_utils/artifact/cli.html#main"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#workflow_utils.artifact.cli.main" title="Permalink to this definition"></a></dt>
<span class="sig-prename descclassname"><span class="pre">workflow_utils.artifact.cli.</span></span><span class="sig-name descname"><span class="pre">main</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">argv</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">['-M',</span> <span class="pre">'html',</span> <span class="pre">'source',</span> <span class="pre">'build',</span> <span class="pre">'-E']</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/workflow_utils/artifact/cli.html#main"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#workflow_utils.artifact.cli.main" title="Permalink to this definition"></a></dt>
<dd></dd></dl>
</section>
......
This diff is collapsed.
......@@ -12,6 +12,22 @@ Subpackages
Submodules
----------
workflow\_utils.call module
---------------------------
.. automodule:: workflow_utils.call
:members:
:undoc-members:
:show-inheritance:
workflow\_utils.conda module
----------------------------
.. automodule:: workflow_utils.conda
:members:
:undoc-members:
:show-inheritance:
workflow\_utils.util module
---------------------------
......
# flake8: noqa
from workflow_utils import artifact
from workflow_utils import call
from workflow_utils import conda
from workflow_utils import util
......@@ -12,16 +12,18 @@ Usage:
call.py 'my.package:callable' arg1 arg2
"""
from typing import List
import argparse
import json
import os
import sys
import re
import importlib
import logging
import subprocess
from glob import glob
from typing import List
log = logging.getLogger('call.py')
......@@ -100,18 +102,13 @@ def python_env_sys_path(env_prefix: str) -> List[str]:
def sys_path_prepend(paths: List[str]):
"""
Appends paths to sys.path and PYTHONPATH.
If you need to foward these sys.path entries along
to a another process (e.g. a spark executor),
you can just make sure that PYTHONPATH is
propagated to it.
Appends paths to sys.path
:param paths:
paths to add to sys.path and PYTHONPATH
paths to add to sys.path
"""
sys.path = paths + sys.path
os.environ['PYTHONPATH'] = ':'.join(paths) + os.environ.get('PYTHONPATH', '')
def include_python_env_sys_path(other_python_env_prefix: str):
......@@ -157,14 +154,75 @@ def import_it(name: str):
symbol_name = None
module = importlib.import_module(module_name)
print(f'Imported {name} as {module.__file__}')
log.info(f'Imported {name} as {module.__file__}')
if symbol_name is None:
return module
return getattr(module, symbol_name)
def call(name: str, *args):
def parse_callable_args(args=[], format='positional'):
"""
Parses list of args into positional and kwargs for
passing to a callable function.
Can parse args in 3 different formats.
- positional (default):
just use args positional args, no kwargs.
- argv
Assume that args is meant to be passed to a CLI
parsing function, e.g. as if it was sys.argv[1:].
In this case, the first positional arg should be
the argv array, so positional_args will be [args]
and kwargs will be empty.
- json
Parse each arg as a json string. If the
arg is a dict, it is assumed to be a kwarg,
else it is a positional args.
Use format=json if you need to pass comprehensive
argument types like ints or lists as args to your
callable.
Returns a tuple of (positional_args, kwargs).
:param args: args to parse
:param format: One of positional, argv, or json.
:return: tuple of (positional_args, kwargs)
"""
positional_args = []
kwargs = {}
if format == 'positional':
positional_args = args
elif format == 'argv':
positional_args=[args]
elif format == 'json':
for arg in args:
# If the arg does not start with a valid json
# opener, assume it is a string value,
# don't bother parsing it.s
if not re.match(r'^["\[\{]', arg):
positional_args += [arg]
else:
parsed_arg = json.loads(arg)
# If the arg is a dict, then assume it is a kwarg.
# This won't work if the callable takes a dict
# as an arg.
if isinstance(parsed_arg, dict):
kwargs.update(parsed_arg)
else:
# Else it is a primitive value, or perhaps a list.
positional_args += [parsed_arg]
return (positional_args, kwargs)
def call(name: str, positional_args, kwargs):
"""
Calls a python function by string name.
......@@ -172,71 +230,91 @@ def call(name: str, *args):
Python module callable name.
E.g. my.package:callable
:param *args: args to pass to callable.
:param positional_args:
positional args to pass to callable.
:param kwargs:
kwargs to pass to callable
:return: result of callable
"""
print(f'Calling {name} with {args}. sys.path: {sys.path}')
log.info(f'Calling {name} with {positional_args}, {kwargs}. sys.path: {sys.path}')
func = import_it(name)
# TO DO: this only works with single var positional args.
# how to parse args if e.g. func takes an array or dict?
# Should we ask user to provide args as json?
return func(args)
return func(*positional_args, **kwargs)
def main(argv=None):
def cli(argv=None):
"""
Imports callable and calls it with args.
If --prefix or --pythonpath are given (e.g. a conda environemnt prefix path),
sys.path and PYTHONPATH_will be altered to include that python environment or pythonpath.
If --prefix are given (e.g. a conda environemnt prefix path),
sys.path will be altered to include that python environment.
This allows us to call a function that might have dependencies in another environment,
effectively 'stacking' the current python env on the base env(s).
effectively 'stacking' the current python env on the other env(s).
--args-format determines how to parse args for the callable. Any args that are not
handled by call.py will be used for the callable.
Example:
::
call.py shutil.which python
call.py --prefix /my/conda/env myproject.transform arg1 arg2
call.py --prefix /my/conda/env --args-format=argv myproject.main --arg1=arg1 --arg2=arg2
call.py --prefix /my/conda/env --args-format=json myproject.fancyfunc \
'["elem1", "elem2"]' '{"kwarg1": "a"}'
"""
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description=main.__doc__)
parser = argparse.ArgumentParser(description=cli.__doc__)
parser.add_argument(
'--prefix',
action='append',
dest='prefixes',
dest='prefix',
type=str,
help='Python environment prefix to include in sys.path.'
help='Python environment prefix to include in sys.path.',
)
parser.add_argument(
'--pythonpath',
action='append',
dest='pythonpaths',
'--log-level',
dest='log_level',
type=str,
help='Extra python paths to include in sys.path.'
help='Log level of call.py logger',
)
parser.add_argument(
'--log-level',
dest='log_level',
'--args-format',
dest='args_format',
type=str,
help='Log level of call.py logger'
help='How to parse args for callable.',
choices=[
'positional',
'argv',
'json',
],
default='positional'
)
parser.add_argument('callable')
parser.add_argument('args', nargs='*')
args = parser.parse_args(argv)
print(args)
# callable_args will be anything not parsed by parser.
(args, callable_args) = parser.parse_known_args(argv)
if args.log_level:
log.setLevel(args.log_level)
if args.prefixes:
for prefix in args.prefixes:
if args.prefix:
for prefix in args.prefix:
include_python_env_sys_path(prefix)
if args.pythonpaths:
sys_path_prepend(args.pythonpaths)
print(f"args:\t{args}\t{','.join(sys.path)}\t{os.environ['PYTHONPATH']}")
(callable_pargs, callable_kwargs) = parse_callable_args(
format=args.args_format,
args=callable_args
)
call(args.callable, *args.args)
log.info(call(args.callable, callable_pargs, callable_kwargs))
if __name__ == '__main__':
main()
cli()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment