refactoring: differentiate between os folders and assigned folders

xtreemfs_client/OSDManager.py

@@ -138,8 +138,8 @@ class OSDManager(object):
         if not div_util.check_for_executable('du'):
             raise ExecutableNotFoundException("No du found. Please make sure it is contained in your PATH.")
 
-        existing_folders = self.get_assigned_folders()
-        managed_folders = []
+        existing_folders = self.get_depth_2_subdirectories()
+        new_folders = []
         for one_folder in existing_folders:
             du = subprocess.run(["du", "-s", one_folder], stdout=subprocess.PIPE,
                                 universal_newlines=True)
@@ -149,9 +149,9 @@ class OSDManager(object):
             new_folder = folder.Folder(self.path_on_volume(one_folder),
                                        folder_size,
                                        None)
-            managed_folders.append(new_folder)
+            new_folders.append(new_folder)
 
-        new_assignments = self.distribution.add_folders(managed_folders)
+        new_assignments = self.distribution.add_folders(new_folders)
 
         if apply_layout:
             self.apply_osd_assignments(new_assignments)
@@ -201,9 +201,8 @@ class OSDManager(object):
         """
         if self.debug:
             print("fixing physical layout externally...")
-        managed_folders = self.get_assigned_folders()
-        for managed_folder in managed_folders:
-            folder_id = self.path_on_volume(managed_folder)
+        managed_folders = self.get_assigned_folder_ids()
+        for folder_id in managed_folders:
             osd_for_folder = self.distribution.get_containing_osd(folder_id)
             self.move_folder_to_osd(folder_id, osd_for_folder.uuid)
 
@@ -232,7 +231,7 @@ class OSDManager(object):
         delete_replica_command_list = []
 
         # create commands
-        managed_folders = self.get_assigned_folders()
+        managed_folders = self.get_depth_2_subdirectories()
         for managed_folder in managed_folders:
             for directory in os.walk(managed_folder):
                 for filename in directory[2]:
@@ -419,6 +418,7 @@ class OSDManager(object):
                                     universal_newlines=True)
                 folder_size = int(du.stdout.split()[0])
 
+            # as the folder_id is generated from the copy source, we cannot call get_path_on_volume to get the foler_id
             new_folder = folder.Folder(os.path.join(self.volume_name, self.path_on_volume, last_2_path_elements),
                                        folder_size,
                                        input_folder)
@@ -475,7 +475,7 @@ class OSDManager(object):
                 host_name = osd_to_host_map[key]
             command = ""
             for move_folder in osd_to_folders_map[key]:
-                folder_path = os.path.join(self.path_to_mount_point, move_folder[len(self.volume_name) + 1:])
+                folder_path = self.get_absolute_file_path(move_folder)
                 folder_tmp_path = os.path.join(tmp_dir, os.path.split(move_folder)[1])
                 # copy folder to temporary location
                 command += "srun -N1-1 --nodelist=" + host_name
@@ -501,7 +501,7 @@ class OSDManager(object):
                                    os.path.split(folder_id)[1])
 
         if tmp_dir is None:
-            tmp_dir = os.path.join(self.path_to_mount_point, self.volume_name, '.tmp_move_folder')
+            tmp_dir = os.path.join(self.path_to_mount_point, '.tmp_move_folder')
 
         start_time = 0
         if self.debug:
@@ -582,7 +582,7 @@ class OSDManager(object):
 
         folders = arg_folders
         if arg_folders is None:
-            folders = self.get_assigned_folders()
+            folders = self.get_depth_2_subdirectories()
 
         for folder_for_update in folders:
             folder_id = self.path_on_volume(folder_for_update)
@@ -789,41 +789,53 @@ class OSDManager(object):
                 print(str(terminated_process.communicate()))
             print("Executing commands done.")
 
-    def get_assigned_folders(self):
+    def get_depth_2_subdirectories(self):
         """
-        creates a list of all folders, managed by this XtreemFS OSD manager, that have an assigned OSD
-        TODO this actually just returns a list of all level 2 subdirectories, independently of whether they are actually
-        assigned to some OSD or not. i think that this is not the expected behaviour.
+        creates a list of all depth 2 subdirectories of self.managed_folder
         """
-        assigned_folders = []
+        subdirectories = []
         for depth_1_folder in os.listdir(self.managed_folder):
             depth_1_path = os.path.join(self.managed_folder, depth_1_folder)
             if os.path.isdir(depth_1_path):
                 for depth_2_folder in os.listdir(depth_1_path):
                     depth_2_path = os.path.join(self.managed_folder, depth_1_folder, depth_2_folder)
                     if os.path.isdir(depth_2_path):
-                        assigned_folders.append(depth_2_path)
+                        subdirectories.append(depth_2_path)
+
+        return subdirectories
 
+    def get_assigned_folder_ids(self):
+        """
+        creates a list of ids of all assigned folders (folders assigned to OSDs)
+        """
+        osd_list = self.distribution.get_osd_list()
+        assigned_folders = []
+        for osd in osd_list:
+            for one_folder in self.distribution.OSDs[osd].folders:
+                assigned_folders.append(one_folder)
         return assigned_folders
 
     def get_target_dir(self, folder_id):
         """
-        gets the path to the target dir, given the folder_id
+        gets the path to the target dir (where to copy the folder), given the folder_id
         """
-        path_on_mount_point = folder_id[len(self.volume_name) + 1:]
-        return os.path.split(os.path.join(self.path_to_mount_point, path_on_mount_point))[0]
+        return os.path.split(self.get_absolute_file_path(folder_id))[0]
 
-    def path_on_volume(self, path):
+    def get_path_on_volume(self, path):
         """
         remove the leading part of the path, such that only the part onto the xtreemfs volume remains, including
         the volume itself.
         throws an exception when the path is not managed by this XtreemFS OSD manager.
+        use this method to calculate the folder_id.
         """
         if not path.startswith(self.managed_folder):
             raise PathNotManagedException("Path " + path + " is not managed by this instance of the XtreemFS OSD"
                                                            "manager!")
         return os.path.join(self.volume_name, path[len(self.path_to_mount_point) + 1:])
 
+    def get_absolute_file_path(self, folder_id):
+        return os.path.join(self.path_to_mount_point, folder_id[len(self.volume_name) + 1:])
+
     def get_containing_folder_id(self, path_on_volume):
         """
         search for the assigned folder that is a prefix of the given path on volume