Commit a3f2c252 authored by Felix Seibert's avatar Felix Seibert
Browse files

introducing optional random osd assignment (instead of LPT assignment)

parent 59384f0c
......@@ -146,7 +146,7 @@ class OSDManager(object):
'''
def copy_folders(self, folders, environment='LOCAL', remote_source=None, sshfs_mount_dir='/tmp/sshfs_tmp_mnt',
apply_layout=True):
apply_layout=True, random_osd_assignment=False):
if not div_util.check_for_executable('du'):
raise ExecutableNotFoundException("No du found. Please make sure it is contained in your PATH.")
......@@ -191,7 +191,7 @@ class OSDManager(object):
if remote_source is not None:
shutil.rmtree(sshfs_mount_dir)
new_assignments = self.distribution.add_folders(new_folders)
new_assignments = self.distribution.add_folders(new_folders, random_osd_assignment)
if apply_layout:
self.apply_osd_assignments(new_assignments)
elif self.debug:
......
......@@ -35,6 +35,7 @@ parser.add_argument("--remote-source", "-r", nargs=1,
help='Use a remote source to copy from. '
'The remote host must be reachable via ssh without password.')
parser.add_argument("--random-layout", "-x", action='store_const', const=True, default=False)
parser.add_argument("--random-osd-assignment", action='store_const', const=True, default=False)
parser.add_argument("--environment", "-e", choices=['LOCAL', 'SLURM', 'HU_CLUSTER'], default='LOCAL')
parser.add_argument("--print", "-p", action='store_const', const=True, default=False,
......@@ -70,7 +71,7 @@ if args.copy:
else:
folders = args.source_folders[0].split(',')
x_man.copy_folders(folders, environment=args.environment, apply_layout=(not args.random_layout),
remote_source=args.remote_source[0])
remote_source=args.remote_source[0], random_osd_assignment=args.random_osd_assignment)
elif args.new_folders:
if len(args.source_folders) == 0:
......
import random
import xtreemfs_client.osd as osd
"""
......@@ -69,7 +71,7 @@ class DataDistribution(object):
if the optional arguments are given, OSDs are assigned data proportionally to their ratio_parameter.
'''
def add_folders(self, folders, osd_information=None, ratio_parameter=''):
def add_folders(self, folders, osd_information=None, ratio_parameter='', random_osd_assignment=False):
new_folders = []
for folder in folders:
containing_osd = self.get_containing_osd(folder.id)
......@@ -78,9 +80,19 @@ class DataDistribution(object):
else:
new_folders.append(folder)
list.sort(new_folders, key=lambda x: x.size, reverse=True)
osds_for_new_folders = []
# totally random OSD assignment, even ignoring OSD capacities
# (might lead to I/O errors when too many groups are assigned to an OSD)
if random_osd_assignment:
for folder in new_folders:
osds_for_new_folders.append((folder.id,
random.choice(self.OSDs.values())))
# balanced deterministic OSD assignment
# (following largest processing time first or post-greedy approach)
list.sort(new_folders, key=lambda x: x.size, reverse=True)
osd_ratios = {}
if osd_information is not None and ratio_parameter != '':
total_osd_size = 0
......@@ -101,7 +113,8 @@ class DataDistribution(object):
<= least_used_osd.totalFolderSize / osd_ratios[least_used_osd.uuid]:
least_used_osd = one_osd
least_used_osd.add_folder(folder.id, folder.size)
osds_for_new_folders.append((folder.id, least_used_osd.uuid))
osds_for_new_folders.append((folder.id,
least_used_osd.uuid))
return osds_for_new_folders
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment