add option to rebalance an existing osd to folder assignment

xtreemfs_client/OSDManager.py

@@ -201,6 +201,69 @@ class OSDManager(object):
             total_time = round(time.time() - start_time)
             print("fixed physical layout of existing files in secs: " + str(total_time))
 
+    def rebalance_existing_assignment(self,
+                                      rebalance_algorithm='lpt',
+                                      fix_layout_internally=True, max_files_in_progress=10000,
+                                      environment='LOCAL',
+                                      movement_strategy='osd_balanced'):
+        if self.debug:
+            print("rebalancing existing distribution... osd manager: \n" + str(self))
+
+        start_time = time.time()
+        self.update()
+        update_time = round(time.time() - start_time)
+        if self.debug:
+            print("updated folder sizes in secs: " + str(update_time))
+
+        start_time = time.time()
+
+        if rebalance_algorithm is 'rebalance_one':
+            movements = self.distribution.rebalance_one_folder()
+        elif rebalance_algorithm is 'two_step_opt':
+            movements = self.distribution.rebalance_two_steps_optimal_matching()
+        elif rebalance_algorithm is 'two_step_rnd':
+            movements = self.distribution.rebalance_two_steps_random_matching()
+        else:
+            movements = self.distribution.rebalance_lpt()
+
+        if self.debug:
+            rebalance_time = round(time.time() - start_time)
+            print("rebalanced assignment in secs: " + str(rebalance_time))
+
+        start_time = time.time()
+
+        if fix_layout_internally:
+            # TODO use movements to make realize_placement more efficient
+            # TODO (for the first calculation of files that need to be moved)
+            placement_realizer = \
+                physicalPlacementRealizer.PhysicalPlacementRealizer(self, debug=self.debug,
+                                                                    max_files_in_progress=max_files_in_progress)
+            placement_realizer.realize_placement(strategy=movement_strategy)
+
+        elif environment == 'SLURM':
+            target_balanced = 1  # 0 is origin balanced, 1 is target balanced
+            # we use self.__generate_move_commands_slurm() to do so.
+            # this either ignores the origin or the target information.
+            # TODO can we do this more intelligently?
+            osd_to_folders_map = {}
+            for folder_to_move in movements.keys():
+                # we have a choice: origin or target balanced.
+                osd_id = movements[folder_to_move][target_balanced]
+                if osd_id in osd_to_folders_map:
+                    osd_to_folders_map[osd_id].append(folder_to_move)
+                else:
+                    osd_to_folders_map[osd_id] = [folder_to_move]
+
+            move_commands = self.__generate_move_commands_slurm(osd_to_folders_map)
+            self.__execute_commands(move_commands)
+
+        else:
+            self.fix_physical_layout_externally()
+
+        if self.debug:
+            total_time = round(time.time() - start_time)
+            print("fixed physical layout of existing files in secs: " + str(total_time))
+
     def fix_physical_layout_externally(self):
         """
         fixes the physical layout, such that it matches the data distribution described in self.distribution.

xtreemfs_client/das.py

@@ -52,6 +52,9 @@ parser.add_argument("--create-from-existing-files", action='store_const', const=
                          'and changes the physical location of files to match the locations prescribed by the '
                          'distribution.')
 
+parser.add_argument("--rebalance-existing-assignment", action='store_const', const=True, default=False,
+                    help='rebalances an existing osd to folder assignment, using the lpt rebalancing method.')
+
 parser.add_argument("--fix-internally", action='store_const', const=True, default=False,
                     help='indicate whether xtreemfs internal functions should be used to fix the physical'
                          'layout. otherwise files will be temporarily located outside xtreemfs,'
@@ -107,3 +110,9 @@ elif args.create_from_existing_files:
                                                   environment=args.environment,
                                                   max_files_in_progress=int(args.max_files_in_progress[0]),
                                                   movement_strategy=args.movement_strategy[0])
+
+elif args.rebalance_existing_assignment:
+    x_man.rebalance_existing_assignment(fix_layout_internally=args.fix_internally,
+                                        environment=args.environment,
+                                        max_files_in_progress=int(args.max_files_in_progress[0]),
+                                        movement_strategy=args.movement_strategy[0])