MultiProcessingPool

class MultiProcessingPool:
    """
    the based Multi Processing Pool Class to be used for lunching FabSim3
    tasks.
    """

    def __init__(self, PoolSize=1):
        try:
            set_start_method("fork")
        except RuntimeError:
            pass
        # to be a little more stable : use one less process maximum
        self.PoolSize = PoolSize if PoolSize < cpu_count() else cpu_count() - 1
        self.Pool = Pool(
            processes=self.PoolSize,
            initializer=start_process,
            initargs=(self.PoolSize,),
        )
        self.Pool_tasks = []

    def add_task(self, func, func_args=dict(), callback_func=None):
        """
        adds the task to create Pool process for execution
        """
        # TODO: it would be better to collect the output results of
        #     each task within a callback function, instead of
        #     iterating over the Pool_tasks
        try:
            self.Pool_tasks.append(
                self.Pool.apply_async(
                    func=func,
                    args=(func_args,),
                    callback=callback_func,
                    error_callback=error_callback,
                )
            )
        except Exception as e:
            self.Pool.close()
            self.Pool.terminate()
            self.Pool.join()
            error_callback(e)
            sys.exit(1)

    def wait_for_tasks(self):
        """
        wait until all tasks in the Pool are finished, then collect the output
        tasks and return the outputs
        """
        results = []
        print("Waiting for tasks to be completed ...")
        # tells the pool not to accept any new job
        self.Pool.close()
        # tells the pool to wait until all jobs finished then exit,
        # effectively cleaning up the pool
        self.Pool.join()

        print("All tasks are finished ...")

        results = [task.get() for task in self.Pool_tasks]

        # make Pool_tasks empty list
        self.Pool_tasks = []

        # Flatten a list of lists
        # source :
        # https://stackoverflow.com/questions/952914/how-to-make-a-flat-list-out-of-list-of-lists
        flatten_results = list(itertools.chain(*results))
        return flatten_results