summaryrefslogtreecommitdiff
path: root/parallelize.py
diff options
context:
space:
mode:
authorScott Gasch <[email protected]>2022-05-31 15:36:40 -0700
committerScott Gasch <[email protected]>2022-05-31 15:36:40 -0700
commit02302bbd9363facb59c4df2c1f4013087702cfa6 (patch)
tree938e51338678416afc891d2839c2ee8dd341f34d /parallelize.py
parentf3dbc7dc19ba5703f8f5aa9bf8af3c491b3510f6 (diff)
Improve docstrings for sphinx.
Diffstat (limited to 'parallelize.py')
-rw-r--r--parallelize.py39
1 files changed, 22 insertions, 17 deletions
diff --git a/parallelize.py b/parallelize.py
index 6005d42..52eb4d1 100644
--- a/parallelize.py
+++ b/parallelize.py
@@ -22,7 +22,10 @@ class Method(Enum):
def parallelize(
_funct: typing.Optional[typing.Callable] = None, *, method: Method = Method.THREAD
) -> typing.Callable:
- """Usage::
+ """This is a decorator that was created to make multi-threading,
+ multi-processing and remote machine parallelism simple in python.
+
+ Sample usage::
@parallelize # defaults to thread-mode
def my_function(a, b, c) -> int:
@@ -43,24 +46,26 @@ def parallelize(
Method.REMOTE: a process on a remote host
The wrapped function returns immediately with a value that is
- wrapped in a SmartFuture. This value will block if it is either
- read directly (via a call to result._resolve) or indirectly (by
- using the result in an expression, printing it, hashing it,
- passing it a function argument, etc...). See comments on the
- SmartFuture class for details.
+ wrapped in a :class:`SmartFuture`. This value will block if it is
+ either read directly (via a call to :meth:`_resolve`) or indirectly
+ (by using the result in an expression, printing it, hashing it,
+ passing it a function argument, etc...). See comments on
+ :class:`SmartFuture` for details.
- Note: you may stack @parallelized methods and it will "work".
- That said, having multiple layers of Method.PROCESS or
- Method.REMOTE may prove to be problematic because each process in
- the stack will use its own independent pool which may overload
- your machine with processes or your network with remote processes
- beyond the control mechanisms built into one instance of the pool.
- Be careful.
+ .. warning::
+ You may stack @parallelized methods and it will "work".
+ That said, having multiple layers of :code:`Method.PROCESS` or
+ :code:`Method.REMOTE` will prove to be problematic because each process in
+ the stack will use its own independent pool which may overload
+ your machine with processes or your network with remote processes
+ beyond the control mechanisms built into one instance of the pool.
+ Be careful.
- Also note: there is a non trivial overhead of pickling code and
- scp'ing it over the network when you use Method.REMOTE. There's
- a smaller but still considerable cost of creating a new process
- and passing code to/from it when you use Method.PROCESS.
+ .. note::
+ There is non-trivial overhead of pickling code and
+ copying it over the network when you use :code:`Method.REMOTE`. There's
+ a smaller but still considerable cost of creating a new process
+ and passing code to/from it when you use :code:`Method.PROCESS`.
"""
def wrapper(funct: typing.Callable):