Commit 59263cf1 authored by Felix Seibert's avatar Felix Seibert
Browse files

using a map instead of a set, to keep track of process starting times

parent 8bf9c502
......@@ -2,6 +2,7 @@ import subprocess
import sys
import socket
import os
import time
def get_osd_uuids(path):
......@@ -53,17 +54,22 @@ execute list of commands in parallel, return list of executions returned with an
def run_commands(commands, max_processes=200, print_errors=True):
running_processes = set()
# running_processes = set()
running_processes_map = {}
errored_processes = []
num_finished = 0
num_total = len(commands)
for command in commands:
started_process = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
running_processes.add(started_process)
if len(running_processes) >= max_processes:
# running_processes.add(started_process)
start_time = time.time()
running_processes_map[started_process] = start_time
# if len(running_processes) >= max_processes:
if len(running_processes_map) >= max_processes:
os.wait()
difference = set()
for running_process in running_processes:
# for running_process in running_processes:
for running_process in running_processes_map.keys():
if running_process.poll() is not None:
num_finished += 1
difference.add(running_process)
......@@ -76,12 +82,16 @@ def run_commands(commands, max_processes=200, print_errors=True):
print_error(finished_process)
print("progress: " + str(num_finished) + "/" + str(num_total))
running_processes = running_processes.difference(difference)
# running_processes = running_processes.difference(difference)
for finished_process in difference:
del running_processes_map[finished_process]
for p in running_processes:
# for p in running_processes:
for p in running_processes_map:
p.wait()
for finished_process in running_processes:
# for finished_process in running_processes:
for finished_process in running_processes_map:
finished_process = (finished_process.args,
finished_process.communicate(),
finished_process.returncode)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment