Bugfixes.

1 files changed, 23 insertions, 17 deletions

diff --git a/executors.py b/executors.py
index ddd62f1..0b4d80e 100644
--- a/executors.py
+++ b/executors.py
@@ -210,6 +210,7 @@ class RemoteWorkerRecord:
 class BundleDetails:
     pickled_code: bytes
     uuid: str
+    fname: str
     worker: Optional[RemoteWorkerRecord]
     username: Optional[str]
     machine: Optional[str]
@@ -586,6 +587,7 @@ class RemoteExecutor(BaseExecutor):
         bundle.worker = worker
         machine = bundle.machine = worker.machine
         username = bundle.username = worker.username
+        fname = bundle.fname
         self.status.record_acquire_worker(worker, uuid)
         logger.debug(f'Running bundle {uuid} on {worker}...')
 
@@ -595,17 +597,17 @@ class RemoteExecutor(BaseExecutor):
                 return self.post_launch_work(bundle)
             except Exception as e:
                 logger.exception(e)
-                logger.info(f"Bundle {uuid} seems to have failed?!")
+                logger.info(f"{uuid}/{fname}: bundle seems to have failed?!")
                 if bundle.failure_count < config.config['executors_max_bundle_failures']:
                     return self.launch(bundle)
                 else:
-                    logger.info(f"Bundle {uuid} is poison, giving up on it.")
+                    logger.info(f"{uuid}/{fname}: bundle is poison, giving up on it.")
                     return None
 
         # Send input to machine if it's not local.
         if hostname not in machine:
             cmd = f'{RSYNC} {bundle.code_file} {username}@{machine}:{bundle.code_file}'
-            logger.info(f"Copying work to {worker} via {cmd}")
+            logger.info(f"{uuid}/{fname}: Copying work to {worker} via {cmd}")
             run_silently(cmd)
 
         # Do it.
@@ -615,7 +617,7 @@ class RemoteExecutor(BaseExecutor):
                f' --code_file {bundle.code_file} --result_file {bundle.result_file}"')
         p = cmd_in_background(cmd, silent=True)
         bundle.pid = pid = p.pid
-        logger.info(f"Running {cmd} in the background as process {pid}")
+        logger.info(f"{uuid}/{fname}: Start training on {worker} via {cmd} (background pid {pid})")
 
         while True:
             try:
@@ -630,7 +632,7 @@ class RemoteExecutor(BaseExecutor):
                     break
             else:
                 logger.debug(
-                    f"{pid}/{bundle.uuid} has finished its work normally."
+                    f"{uuid}/{fname}: pid {pid} has finished its work normally."
                 )
                 break
 
@@ -638,10 +640,10 @@ class RemoteExecutor(BaseExecutor):
             return self.post_launch_work(bundle)
         except Exception as e:
             logger.exception(e)
-            logger.info(f"Bundle {uuid} seems to have failed?!")
+            logger.info(f"{uuid}: Bundle seems to have failed?!")
             if bundle.failure_count < config.config['executors_max_bundle_failures']:
                 return self.launch(bundle)
-            logger.info(f"Bundle {uuid} is poison, giving up on it.")
+            logger.info(f"{uuid}: Bundle is poison, giving up on it.")
             return None
 
     def post_launch_work(self, bundle: BundleDetails) -> Any:
@@ -652,6 +654,8 @@ class RemoteExecutor(BaseExecutor):
             machine = bundle.machine
             result_file = bundle.result_file
             code_file = bundle.code_file
+            fname = bundle.fname
+            uuid = bundle.uuid
 
             # Whether original or backup, if we finished first we must
             # fetch the results if the computation happened on a
@@ -662,7 +666,7 @@ class RemoteExecutor(BaseExecutor):
                 if bundle.hostname not in bundle.machine:
                     cmd = f'{RSYNC} {username}@{machine}:{result_file} {result_file} 2>/dev/null'
                     logger.info(
-                        f"Fetching results from {username}@{machine} via {cmd}"
+                        f"{uuid}/{fname}: Fetching results from {username}@{machine} via {cmd}"
                     )
                     try:
                         run_silently(cmd)
@@ -686,7 +690,7 @@ class RemoteExecutor(BaseExecutor):
         # if one of the backups finished first; it still must read the
         # result from disk.
         if is_original:
-            logger.debug(f"Unpickling {result_file}.")
+            logger.debug(f"{uuid}/{fname}: Unpickling {result_file}.")
             try:
                 with open(f'{result_file}', 'rb') as rb:
                     serialized = rb.read()
@@ -707,7 +711,7 @@ class RemoteExecutor(BaseExecutor):
             if bundle.backup_bundles is not None:
                 for backup in bundle.backup_bundles:
                     logger.debug(
-                        f'Notifying backup {backup.uuid} that it is cancelled'
+                        f'{uuid}/{fname}: Notifying backup {backup.uuid} that it\'s cancelled'
                     )
                     backup.is_cancelled.set()
 
@@ -721,7 +725,7 @@ class RemoteExecutor(BaseExecutor):
             # Tell the original to stop if we finished first.
             if not was_cancelled:
                 logger.debug(
-                    f'Notifying original {bundle.src_bundle.uuid} that it is cancelled'
+                    f'{uuid}/{fname}: Notifying original {bundle.src_bundle.uuid} that it\'s cancelled'
                 )
                 bundle.src_bundle.is_cancelled.set()
 
@@ -730,7 +734,7 @@ class RemoteExecutor(BaseExecutor):
         self.adjust_task_count(-1)
         return result
 
-    def create_original_bundle(self, pickle):
+    def create_original_bundle(self, pickle, fname: str):
         from string_utils import generate_uuid
         uuid = generate_uuid(as_hex=True)
         code_file = f'/tmp/{uuid}.code.bin'
@@ -743,6 +747,7 @@ class RemoteExecutor(BaseExecutor):
         bundle = BundleDetails(
             pickled_code = pickle,
             uuid = uuid,
+            fname = fname,
             worker = None,
             username = None,
             machine = None,
@@ -761,7 +766,7 @@ class RemoteExecutor(BaseExecutor):
             failure_count = 0,
         )
         self.status.record_bundle_details(bundle)
-        logger.debug(f'Created original bundle {uuid}')
+        logger.debug(f'{uuid}/{fname}: Created original bundle')
         return bundle
 
     def create_backup_bundle(self, src_bundle: BundleDetails):
@@ -772,6 +777,7 @@ class RemoteExecutor(BaseExecutor):
         backup_bundle = BundleDetails(
             pickled_code = src_bundle.pickled_code,
             uuid = uuid,
+            fname = src_bundle.fname,
             worker = None,
             username = None,
             machine = None,
@@ -791,7 +797,7 @@ class RemoteExecutor(BaseExecutor):
         )
         src_bundle.backup_bundles.append(backup_bundle)
         self.status.record_bundle_details_already_locked(backup_bundle)
-        logger.debug(f'Created backup bundle {uuid}')
+        logger.debug(f'{uuid}/{src_bundle.fname}: Created backup bundle')
         return backup_bundle
 
     def schedule_backup_for_bundle(self,
@@ -799,7 +805,7 @@ class RemoteExecutor(BaseExecutor):
         assert self.status.lock.locked()
         backup_bundle = self.create_backup_bundle(src_bundle)
         logger.debug(
-            f'Scheduling backup bundle {backup_bundle.uuid} for execution'
+            f'{backup_bundle.uuid}/{backup_bundle.fname}: Scheduling backup for execution...'
         )
         self._helper_executor.submit(self.launch, backup_bundle)
 
@@ -812,7 +818,7 @@ class RemoteExecutor(BaseExecutor):
                *args,
                **kwargs) -> fut.Future:
         pickle = make_cloud_pickle(function, *args, **kwargs)
-        bundle = self.create_original_bundle(pickle)
+        bundle = self.create_original_bundle(pickle, function.__name__)
         self.total_bundles_submitted += 1
         return self._helper_executor.submit(self.launch, bundle)
 
@@ -882,7 +888,7 @@ class DefaultExecutors(object):
                     RemoteWorkerRecord(
                         username = 'scott',
                         machine = 'meerkat.cabin',
-                        weight = 6,
+                        weight = 5,
                         count = 2,
                     ),
                 )