Commit a488b8c0 authored by Felix Seibert's avatar Felix Seibert
Browse files

add option to rebalance an existing osd to folder assignment

parent e86e345d
......@@ -201,6 +201,69 @@ class OSDManager(object):
total_time = round(time.time() - start_time)
print("fixed physical layout of existing files in secs: " + str(total_time))
def rebalance_existing_assignment(self,
rebalance_algorithm='lpt',
fix_layout_internally=True, max_files_in_progress=10000,
environment='LOCAL',
movement_strategy='osd_balanced'):
if self.debug:
print("rebalancing existing distribution... osd manager: \n" + str(self))
start_time = time.time()
self.update()
update_time = round(time.time() - start_time)
if self.debug:
print("updated folder sizes in secs: " + str(update_time))
start_time = time.time()
if rebalance_algorithm is 'rebalance_one':
movements = self.distribution.rebalance_one_folder()
elif rebalance_algorithm is 'two_step_opt':
movements = self.distribution.rebalance_two_steps_optimal_matching()
elif rebalance_algorithm is 'two_step_rnd':
movements = self.distribution.rebalance_two_steps_random_matching()
else:
movements = self.distribution.rebalance_lpt()
if self.debug:
rebalance_time = round(time.time() - start_time)
print("rebalanced assignment in secs: " + str(rebalance_time))
start_time = time.time()
if fix_layout_internally:
# TODO use movements to make realize_placement more efficient
# TODO (for the first calculation of files that need to be moved)
placement_realizer = \
physicalPlacementRealizer.PhysicalPlacementRealizer(self, debug=self.debug,
max_files_in_progress=max_files_in_progress)
placement_realizer.realize_placement(strategy=movement_strategy)
elif environment == 'SLURM':
target_balanced = 1 # 0 is origin balanced, 1 is target balanced
# we use self.__generate_move_commands_slurm() to do so.
# this either ignores the origin or the target information.
# TODO can we do this more intelligently?
osd_to_folders_map = {}
for folder_to_move in movements.keys():
# we have a choice: origin or target balanced.
osd_id = movements[folder_to_move][target_balanced]
if osd_id in osd_to_folders_map:
osd_to_folders_map[osd_id].append(folder_to_move)
else:
osd_to_folders_map[osd_id] = [folder_to_move]
move_commands = self.__generate_move_commands_slurm(osd_to_folders_map)
self.__execute_commands(move_commands)
else:
self.fix_physical_layout_externally()
if self.debug:
total_time = round(time.time() - start_time)
print("fixed physical layout of existing files in secs: " + str(total_time))
def fix_physical_layout_externally(self):
"""
fixes the physical layout, such that it matches the data distribution described in self.distribution.
......
......@@ -52,6 +52,9 @@ parser.add_argument("--create-from-existing-files", action='store_const', const=
'and changes the physical location of files to match the locations prescribed by the '
'distribution.')
parser.add_argument("--rebalance-existing-assignment", action='store_const', const=True, default=False,
help='rebalances an existing osd to folder assignment, using the lpt rebalancing method.')
parser.add_argument("--fix-internally", action='store_const', const=True, default=False,
help='indicate whether xtreemfs internal functions should be used to fix the physical'
'layout. otherwise files will be temporarily located outside xtreemfs,'
......@@ -107,3 +110,9 @@ elif args.create_from_existing_files:
environment=args.environment,
max_files_in_progress=int(args.max_files_in_progress[0]),
movement_strategy=args.movement_strategy[0])
elif args.rebalance_existing_assignment:
x_man.rebalance_existing_assignment(fix_layout_internally=args.fix_internally,
environment=args.environment,
max_files_in_progress=int(args.max_files_in_progress[0]),
movement_strategy=args.movement_strategy[0])
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment