dataDistribution.py

import random

import xtreemfs_client.osd as osd

"""
class to keep track of the osd (object storage device) locations of different folders, i.e.,
their physical location.

this class also allows to calculate a 'good' osd for new data, based on the
distribution known beforehand.
"""


class DataDistribution(object):
    def __init__(self):
        self.OSDs = {}

    def add_new_osd(self, osd_uuid):
        if osd_uuid in self.OSDs:
            print("key: " + osd_uuid + " is already present!")
            return
        new_osd = osd.OSD(osd_uuid)
        self.OSDs[osd_uuid] = new_osd

    def add_osd(self, new_osd):
        if new_osd.uuid in self.OSDs:
            print("key: " + new_osd.uuid + " is already present!")
            return
        self.OSDs[new_osd.uuid] = new_osd

    def add_osd_list(self, osd_list):
        for osdUuid in osd_list:
            if osdUuid not in self.OSDs:
                new_osd = osd.OSD(osdUuid)
                self.OSDs[osdUuid] = new_osd

    def get_osd_list(self):
        osd_list = []
        for osd_name in self.OSDs.keys():
            osd_list.append(osd_name)
        return osd_list

    def get_containing_osd(self, folder_id):
        for checked_osd in self.OSDs.values():
            if checked_osd.contains_folder(folder_id):
                return checked_osd
        return None

    def get_average_folder_size(self):
        total_size = 0
        total_number_of_folders = 0
        for one_osd in self.OSDs.values():
            total_size += one_osd.totalFolderSize
            total_number_of_folders += len(one_osd.folders)
        if total_number_of_folders == 0:
            return 0
        return total_size / total_number_of_folders

    def assign_new_osd(self, folder_id, new_osd):
        old_osd = self.get_containing_osd(folder_id)
        if old_osd is None:
            self.OSDs[new_osd].add_folder(folder_id, self.get_average_folder_size())
        else:
            self.OSDs[new_osd].add_folder(folder_id, self.OSDs[old_osd.uuid].folders[folder_id])
            self.OSDs[old_osd.uuid].remove_folder(folder_id)

    '''
    adds a list of folders to the data distribution.
    returns a list of assignments from folders to OSDs, for which (folders) there was previously no assignment.

    if the optional arguments are given, OSDs are assigned data proportionally to their ratio_parameter.
    '''

    def add_folders(self, folders, osd_information=None, ratio_parameter='', random_osd_assignment=False):
        new_folders = []
        for folder in folders:
            containing_osd = self.get_containing_osd(folder.id)
            if containing_osd is not None:
                containing_osd.add_folder(folder.id, folder.size)
            else:
                new_folders.append(folder)

        osds_for_new_folders = []

        # totally random OSD assignment, even ignoring OSD capacities
        # (might lead to I/O errors when too many groups are assigned to an OSD)
        if random_osd_assignment:
            for folder in new_folders:
                osds_for_new_folders.append((folder.id,
                                             random.choice(self.OSDs.values())))

        # balanced deterministic OSD assignment
        # (following largest processing time first or post-greedy approach)
        list.sort(new_folders, key=lambda x: x.size, reverse=True)

        osd_ratios = {}
        if osd_information is not None and ratio_parameter != '':
            total_osd_size = 0
            for osd_size in osd_information.values():
                total_osd_size += osd_size[ratio_parameter]
            for osd_uuid, osd_size in osd_information.items():
                osd_ratios[osd_uuid] = float(osd_size[ratio_parameter]) / float(total_osd_size)

        else:
            for osd_uuid in self.OSDs.keys():
                osd_ratios[osd_uuid] = float(1)

        for folder in new_folders:
            least_used_osd = None
            for one_osd in self.OSDs.values():
                if (least_used_osd is None) or \
                                        one_osd.totalFolderSize / osd_ratios[one_osd.uuid] \
                                <= least_used_osd.totalFolderSize / osd_ratios[least_used_osd.uuid]:
                    least_used_osd = one_osd
            least_used_osd.add_folder(folder.id, folder.size)
            osds_for_new_folders.append((folder.id,
                                         least_used_osd.uuid))

        return osds_for_new_folders

    '''
    updates the size of a given folder
    '''

    def update_folder(self, folder, size):
        for one_osd in self.OSDs.values():
            if folder in one_osd.folders.keys():
                one_osd.update_folder(folder, size)

    '''
    generates a string describing this data distribution
    '''

    def description(self):
        string = ""
        for one_osd in self.OSDs.values():
            string += str(one_osd)
            string += "\n"
            string += "folders : " + str(one_osd.folders)
            string += "\n"
        string += "average folder size: " + str(self.get_average_folder_size())
        return string

    def __str__(self):
        string_representation = "DataDistribution has " + str(len(self.OSDs)) \
                                + " osds: \n"
        for key, value in self.OSDs.items():
            string_representation += str(value) + " \n"
        return string_representation