diff --git a/sergeant/supervisor.py b/sergeant/supervisor.py index fd9b8dd..b01aac4 100644 --- a/sergeant/supervisor.py +++ b/sergeant/supervisor.py @@ -99,6 +99,8 @@ def __init__( self.concurrent_workers = concurrent_workers self.max_worker_memory_usage = max_worker_memory_usage + self.supevisor_process = psutil.Process() + self.extra_signature: typing.Dict[str, typing.Any] = { 'supervisor': { 'worker_module_name': self.worker_module_name, @@ -154,6 +156,8 @@ def supervise_loop( worker=worker, ) + self.clean_zombies() + time.sleep(0.5) self.logger.info( @@ -288,6 +292,7 @@ def respawn_a_worker( worker: SupervisedWorker, ) -> None: worker.kill() + self.current_workers.remove(worker) new_worker = SupervisedWorker( worker_module_name=self.worker_module_name, @@ -304,12 +309,31 @@ def stop_a_worker( worker: SupervisedWorker, ) -> None: worker.kill() + self.current_workers.remove(worker) self.logger.info( msg=f'worker has stopped: {worker.process.pid}', extra=self.extra_signature, ) + def clean_zombies( + self, + ): + supervisor_zombie_children = [ + child_process + for child_process in self.supevisor_process.children() + if child_process.status() == psutil.STATUS_ZOMBIE + ] + if supervisor_zombie_children: + self.logger.info( + msg=f'cleaning {len(supervisor_zombie_children)} zombies', + extra=self.extra_signature, + ) + psutil.wait_procs( + procs=supervisor_zombie_children, + timeout=1.0, + ) + def main() -> None: parser = argparse.ArgumentParser( diff --git a/setup.py b/setup.py index 2376a53..a21e9c2 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setuptools.setup( name='sergeant', - version='0.18.1', + version='0.18.2', author='Gal Ben David', author_email='gal@intsights.com', url='https://github.com/Intsights/sergeant',