Add BlockIdManagerSelector

docs/reference.rst

@@ -78,6 +78,16 @@ Executors
     parsl.executors.FluxExecutor
     parsl.executors.radical.RadicalPilotExecutor
 
+Executors Miscellaneous
+=======================
+
+.. autosummary::
+    :toctree: stubs
+    :nosignatures:
+
+    parsl.executors.high_throughput.manager_selector.RandomManagerSelector
+    parsl.executors.high_throughput.manager_selector.BlockIdManagerSelector
+
 Launchers
 =========
 

parsl/executors/high_throughput/executor.py

@@ -145,6 +145,10 @@
 
     encrypted : bool
         Flag to enable/disable encryption (CurveZMQ). Default is False.
+
+    manager_selector: ManagerSelector
+        Determines what strategy the interchange uses to select managers during task distribution.
+        See API reference under "Executors Miscellaneous" regarding the various manager selectors.
 """  # Documentation for params used by both HTEx and MPIEx
 
 

parsl/executors/high_throughput/manager_selector.py

@@ -19,6 +19,14 @@ def sort_managers(self, ready_managers: Dict[bytes, ManagerRecord], manager_list
 
 class RandomManagerSelector(ManagerSelector):
 
+    """Returns a shuffled list of interesting_managers
+
+    Maintains the behavior of the original interchange. Works well
+    in distributing workloads equally across all availble compute
+    resources. Is not effective in conjunction with elastic scaling
+    behavior as it leads to wasted resource consumption.
+    """
+
     def sort_managers(self, ready_managers: Dict[bytes, ManagerRecord], manager_list: Set[bytes]) -> List[bytes]:
         c_manager_list = list(manager_list)
         random.shuffle(c_manager_list)
@@ -27,5 +35,19 @@ def sort_managers(self, ready_managers: Dict[bytes, ManagerRecord], manager_list
 
 class BlockIdManagerSelector(ManagerSelector):
 
+    """Returns a interesting_managers list sorted by block ID
+
+    Observations:
+    1. BlockID manager selector helps with workloads that see a varying
+    amount of tasks over time. We see new blocks are prioritized with the
+    blockID manager selector, when used with 'htex_auto_scaling', results
+    in compute cost savings.
+
+    2. Doesn't really work with bag-of-tasks workloads. When all the tasks
+    are put into the queue upfront, all blocks operate at near full
+    utilization for the majority of the workload, which task goes where
+    doesn't really matter.
+    """
+
     def sort_managers(self, ready_managers: Dict[bytes, ManagerRecord], manager_list: Set[bytes]) -> List[bytes]:
         return sorted(manager_list, key=lambda x: (ready_managers[x]['block_id'] is not None, ready_managers[x]['block_id']), reverse=True)