import xtreemfs_client.osd as osd """ class to keep track of the osd (object storage device) locations of different folders, i.e., their physical location. this class also allows to calculate a 'good' osd for new data, based on the distribution known beforehand. """ class DataDistribution(object): def __init__(self): self.OSDs = {} def add_new_osd(self, osd_uuid): if osd_uuid in self.OSDs: print("key: " + osd_uuid + " is already present!") return new_osd = osd.OSD(osd_uuid) self.OSDs[osd_uuid] = new_osd def add_osd(self, new_osd): if new_osd.uuid in self.OSDs: print("key: " + new_osd.uuid + " is already present!") return self.OSDs[new_osd.uuid] = new_osd def add_osd_list(self, osd_list): for osdUuid in osd_list: if osdUuid not in self.OSDs: new_osd = osd.OSD(osdUuid) self.OSDs[osdUuid] = new_osd def get_osd_list(self): osd_list = [] for osd_name in self.OSDs.keys(): osd_list.append(osd_name) return osd_list def get_containing_osd(self, folder_id): for checked_osd in self.OSDs.values(): if checked_osd.contains_folder(folder_id): return checked_osd return None def get_average_folder_size(self): total_size = 0 total_number_of_folders = 0 for one_osd in self.OSDs.values(): total_size += one_osd.totalFolderSize total_number_of_folders += len(one_osd.folders) if total_number_of_folders == 0: return 0 return total_size / total_number_of_folders def assign_new_osd(self, folder_id, new_osd): old_osd = self.get_containing_osd(folder_id) if old_osd is None: self.OSDs[new_osd].add_folder(folder_id, self.get_average_folder_size()) else: self.OSDs[new_osd].add_folder(folder_id, self.OSDs[old_osd.uuid].folders[folder_id]) self.OSDs[old_osd.uuid].remove_folder(folder_id) ''' adds a list of folders to the data distribution. returns a list of assignments from folders to OSDs, for which (folders) there was previously no assignment. if the optional arguments are given, OSDs are assigned data proportionally to their ratio_parameter. ''' def add_folders(self, folders, osd_information=None, ratio_parameter=''): new_folders = [] for folder in folders: containing_osd = self.get_containing_osd(folder.id) if containing_osd is not None: containing_osd.add_folder(folder.id, folder.size) else: new_folders.append(folder) list.sort(new_folders, key=lambda x: x.size, reverse=True) osds_for_new_folders = [] osd_ratios = {} if osd_information is not None and ratio_parameter != '': total_osd_size = 0 for osd_size in osd_information.values(): total_osd_size += osd_size[ratio_parameter] for osd_uuid, osd_size in osd_information.items(): osd_ratios[osd_uuid] = float(osd_size[ratio_parameter]) / float(total_osd_size) else: for osd_uuid in self.OSDs.keys(): osd_ratios[osd_uuid] = float(1) for folder in new_folders: least_used_osd = None for one_osd in self.OSDs.values(): if (least_used_osd is None) or \ one_osd.totalFolderSize / osd_ratios[one_osd.uuid] \ <= least_used_osd.totalFolderSize / osd_ratios[least_used_osd.uuid]: least_used_osd = one_osd least_used_osd.add_folder(folder.id, folder.size) osds_for_new_folders.append((folder.id, least_used_osd.uuid)) return osds_for_new_folders ''' updates the size of a given folder ''' def update_folder(self, folder, size): for one_osd in self.OSDs.values(): if folder in one_osd.folders.keys(): one_osd.update_folder(folder, size) ''' generates a string describing this data distribution ''' def description(self): string = "" for one_osd in self.OSDs.values(): string += str(one_osd) string += "\n" string += "folders : " + str(one_osd.folders) string += "\n" string += "average folder size: " + str(self.get_average_folder_size()) return string def __str__(self): string_representation = "DataDistribution has " + str(len(self.OSDs)) \ + " osds: \n" for key, value in self.OSDs.items(): string_representation += str(value) + " \n" return string_representation