diff options
Diffstat (limited to 'parallelize.py')
| -rw-r--r-- | parallelize.py | 39 |
1 files changed, 22 insertions, 17 deletions
diff --git a/parallelize.py b/parallelize.py index 6005d42..52eb4d1 100644 --- a/parallelize.py +++ b/parallelize.py @@ -22,7 +22,10 @@ class Method(Enum): def parallelize( _funct: typing.Optional[typing.Callable] = None, *, method: Method = Method.THREAD ) -> typing.Callable: - """Usage:: + """This is a decorator that was created to make multi-threading, + multi-processing and remote machine parallelism simple in python. + + Sample usage:: @parallelize # defaults to thread-mode def my_function(a, b, c) -> int: @@ -43,24 +46,26 @@ def parallelize( Method.REMOTE: a process on a remote host The wrapped function returns immediately with a value that is - wrapped in a SmartFuture. This value will block if it is either - read directly (via a call to result._resolve) or indirectly (by - using the result in an expression, printing it, hashing it, - passing it a function argument, etc...). See comments on the - SmartFuture class for details. + wrapped in a :class:`SmartFuture`. This value will block if it is + either read directly (via a call to :meth:`_resolve`) or indirectly + (by using the result in an expression, printing it, hashing it, + passing it a function argument, etc...). See comments on + :class:`SmartFuture` for details. - Note: you may stack @parallelized methods and it will "work". - That said, having multiple layers of Method.PROCESS or - Method.REMOTE may prove to be problematic because each process in - the stack will use its own independent pool which may overload - your machine with processes or your network with remote processes - beyond the control mechanisms built into one instance of the pool. - Be careful. + .. warning:: + You may stack @parallelized methods and it will "work". + That said, having multiple layers of :code:`Method.PROCESS` or + :code:`Method.REMOTE` will prove to be problematic because each process in + the stack will use its own independent pool which may overload + your machine with processes or your network with remote processes + beyond the control mechanisms built into one instance of the pool. + Be careful. - Also note: there is a non trivial overhead of pickling code and - scp'ing it over the network when you use Method.REMOTE. There's - a smaller but still considerable cost of creating a new process - and passing code to/from it when you use Method.PROCESS. + .. note:: + There is non-trivial overhead of pickling code and + copying it over the network when you use :code:`Method.REMOTE`. There's + a smaller but still considerable cost of creating a new process + and passing code to/from it when you use :code:`Method.PROCESS`. """ def wrapper(funct: typing.Callable): |