handling processes that do not terminate even after they got killed

xtreemfs_client/div_util.py

@@ -86,6 +86,8 @@ def run_commands(commands, max_processes=200, print_errors=True):
             for finished_process in difference:
                 del running_processes_map[finished_process]
 
+
+    timeout_processes = 0
     # for finished_process in running_processes:
     for still_running_process in running_processes_map:
         try:
@@ -94,7 +96,11 @@ def run_commands(commands, max_processes=200, print_errors=True):
             print("process timed out: " + str(still_running_process.args))
             still_running_process.kill()
             print("process killed.")
-            stdout, stderr = still_running_process.communicate(timeout=10)
+            try:
+                stdout, stderr = still_running_process.communicate(timeout=10)
+            except subprocess.TimeoutExpired:
+                timeout_processes += 1
+                continue
 
         still_running_process = (still_running_process.args,
                             (stdout, stderr),
@@ -107,6 +113,9 @@ def run_commands(commands, max_processes=200, print_errors=True):
         num_finished += 1
         print("progress: " + str(num_finished) + "/" + str(num_total))
 
+    if timeout_processes > 0:
+        print("number of processes with expired timeout: " + str(timeout_processes))
+
     return errored_processes
     # return list(map(lambda proc: (proc.args, proc.communicate(), proc.returncode), all_processes))