Commit a488b8c0 authored by Felix Seibert's avatar Felix Seibert
Browse files

add option to rebalance an existing osd to folder assignment

parent e86e345d
...@@ -201,6 +201,69 @@ class OSDManager(object): ...@@ -201,6 +201,69 @@ class OSDManager(object):
total_time = round(time.time() - start_time) total_time = round(time.time() - start_time)
print("fixed physical layout of existing files in secs: " + str(total_time)) print("fixed physical layout of existing files in secs: " + str(total_time))
def rebalance_existing_assignment(self,
rebalance_algorithm='lpt',
fix_layout_internally=True, max_files_in_progress=10000,
environment='LOCAL',
movement_strategy='osd_balanced'):
if self.debug:
print("rebalancing existing distribution... osd manager: \n" + str(self))
start_time = time.time()
self.update()
update_time = round(time.time() - start_time)
if self.debug:
print("updated folder sizes in secs: " + str(update_time))
start_time = time.time()
if rebalance_algorithm is 'rebalance_one':
movements = self.distribution.rebalance_one_folder()
elif rebalance_algorithm is 'two_step_opt':
movements = self.distribution.rebalance_two_steps_optimal_matching()
elif rebalance_algorithm is 'two_step_rnd':
movements = self.distribution.rebalance_two_steps_random_matching()
else:
movements = self.distribution.rebalance_lpt()
if self.debug:
rebalance_time = round(time.time() - start_time)
print("rebalanced assignment in secs: " + str(rebalance_time))
start_time = time.time()
if fix_layout_internally:
# TODO use movements to make realize_placement more efficient
# TODO (for the first calculation of files that need to be moved)
placement_realizer = \
physicalPlacementRealizer.PhysicalPlacementRealizer(self, debug=self.debug,
max_files_in_progress=max_files_in_progress)
placement_realizer.realize_placement(strategy=movement_strategy)
elif environment == 'SLURM':
target_balanced = 1 # 0 is origin balanced, 1 is target balanced
# we use self.__generate_move_commands_slurm() to do so.
# this either ignores the origin or the target information.
# TODO can we do this more intelligently?
osd_to_folders_map = {}
for folder_to_move in movements.keys():
# we have a choice: origin or target balanced.
osd_id = movements[folder_to_move][target_balanced]
if osd_id in osd_to_folders_map:
osd_to_folders_map[osd_id].append(folder_to_move)
else:
osd_to_folders_map[osd_id] = [folder_to_move]
move_commands = self.__generate_move_commands_slurm(osd_to_folders_map)
self.__execute_commands(move_commands)
else:
self.fix_physical_layout_externally()
if self.debug:
total_time = round(time.time() - start_time)
print("fixed physical layout of existing files in secs: " + str(total_time))
def fix_physical_layout_externally(self): def fix_physical_layout_externally(self):
""" """
fixes the physical layout, such that it matches the data distribution described in self.distribution. fixes the physical layout, such that it matches the data distribution described in self.distribution.
......
...@@ -52,6 +52,9 @@ parser.add_argument("--create-from-existing-files", action='store_const', const= ...@@ -52,6 +52,9 @@ parser.add_argument("--create-from-existing-files", action='store_const', const=
'and changes the physical location of files to match the locations prescribed by the ' 'and changes the physical location of files to match the locations prescribed by the '
'distribution.') 'distribution.')
parser.add_argument("--rebalance-existing-assignment", action='store_const', const=True, default=False,
help='rebalances an existing osd to folder assignment, using the lpt rebalancing method.')
parser.add_argument("--fix-internally", action='store_const', const=True, default=False, parser.add_argument("--fix-internally", action='store_const', const=True, default=False,
help='indicate whether xtreemfs internal functions should be used to fix the physical' help='indicate whether xtreemfs internal functions should be used to fix the physical'
'layout. otherwise files will be temporarily located outside xtreemfs,' 'layout. otherwise files will be temporarily located outside xtreemfs,'
...@@ -107,3 +110,9 @@ elif args.create_from_existing_files: ...@@ -107,3 +110,9 @@ elif args.create_from_existing_files:
environment=args.environment, environment=args.environment,
max_files_in_progress=int(args.max_files_in_progress[0]), max_files_in_progress=int(args.max_files_in_progress[0]),
movement_strategy=args.movement_strategy[0]) movement_strategy=args.movement_strategy[0])
elif args.rebalance_existing_assignment:
x_man.rebalance_existing_assignment(fix_layout_internally=args.fix_internally,
environment=args.environment,
max_files_in_progress=int(args.max_files_in_progress[0]),
movement_strategy=args.movement_strategy[0])
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment