Commit 6d7ea2ac authored by Felix Seibert's avatar Felix Seibert
Browse files

introducing more python-like variable naming

parent 7d9cc389
......@@ -30,8 +30,8 @@ max_processes_delete_replica = 200
class OSDManager(object):
def __init__(self, path_to_managed_folder, config_file='.das_config', value_map=None, debug=False):
self.managedFolder = path_to_managed_folder
self.configFile = config_file
self.managed_folder = path_to_managed_folder
self.config_file = config_file
self.debug = debug
if value_map is None:
......@@ -39,7 +39,7 @@ class OSDManager(object):
if not div_util.check_for_executable('xtfsutil'):
raise ExecutableNotFoundException("No xtfsutil found. Please make sure it is contained in your PATH.")
output_1 = subprocess.run(["xtfsutil", self.managedFolder], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
output_1 = subprocess.run(["xtfsutil", self.managed_folder], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
if output_1.stderr.startswith("xtfsutil failed: Path doesn't point to an entity on an XtreemFS volume!"):
......@@ -49,18 +49,18 @@ class OSDManager(object):
if len(output_1.stderr) > 0:
raise Exception("xtfsutil produced some error: " + output_1.stderr)
self.pathOnVolume = div_util.remove_leading_trailing_slashes(
self.path_on_volume = div_util.remove_leading_trailing_slashes(
str(output_1.stdout).split("\n")[0].split()[-1])
self.pathToMountPoint = self.managedFolder[0:(len(self.managedFolder) - len(self.pathOnVolume) - 1)]
self.path_to_mount_point = self.managed_folder[0:(len(self.managed_folder) - len(self.path_on_volume) - 1)]
output_2 = subprocess.run(["xtfsutil", self.pathToMountPoint],
output_2 = subprocess.run(["xtfsutil", self.path_to_mount_point],
stdout=subprocess.PIPE, universal_newlines=True)
self.volume_information = div_util.extract_volume_information(output_2.stdout)
self.volumeName = self.volume_information[0]
self.volume_name = self.volume_information[0]
osd_list = list(map(lambda x: x[0], self.volume_information[1]))
self.osdSelectionPolicy = self.volume_information[2]
self.volumeAddress = self.volume_information[3]
self.osd_selection_policy = self.volume_information[2]
self.volume_address = self.volume_information[3]
self.distribution = None
if not self.__read_configuration():
......@@ -68,10 +68,10 @@ class OSDManager(object):
self.distribution.add_osd_list(osd_list)
self.osdInformation = None
self.osd_information = None
try:
answer = request.urlopen(div_util.get_http_address(self.volumeAddress))
answer = request.urlopen(div_util.get_http_address(self.volume_address))
html_data = answer.read().decode('UTF-8')
parser = dir_status_page_parser.dir_status_page_parser()
......@@ -81,7 +81,7 @@ class OSDManager(object):
filtered_data_sets = list(filter(lambda x: int(x['last updated'].split()[0]) != 0, parser.dataSets))
filtered_data_sets = list(filter(lambda x: x['type'] == 'SERVICE_TYPE_OSD', filtered_data_sets))
self.osdInformation = {}
self.osd_information = {}
for data_set in filtered_data_sets:
uuid = data_set['uuid']
......@@ -89,28 +89,28 @@ class OSDManager(object):
current_osd['usable_space'] = int(data_set['usable'].split()[0])
current_osd['total_space'] = int(data_set['total'].split()[0])
self.osdInformation[uuid] = current_osd
self.osd_information[uuid] = current_osd
except urllib.error.URLError as error:
print("osd information could not be fetched! Probably the http status page could not be found at:",
div_util.get_http_address(self.volumeAddress))
div_util.get_http_address(self.volume_address))
print(error)
else:
try:
self.pathOnVolume = value_map['path_on_volume']
self.pathToMountPoint = value_map['path_to_mount']
self.volumeName = value_map['volume_name']
self.osdSelectionPolicy = value_map['osd_selection_policy']
self.path_on_volume = value_map['path_on_volume']
self.path_to_mount_point = value_map['path_to_mount']
self.volume_name = value_map['volume_name']
self.osd_selection_policy = value_map['osd_selection_policy']
self.distribution = value_map['data_distribution']
self.volumeAddress = value_map['volume_address']
self.osdInformation = value_map['osd_information']
self.volume_address = value_map['volume_address']
self.osd_information = value_map['osd_information']
except KeyError as error:
print('key not found:', error)
print('leaving in OSDManager field empty!')
def __read_configuration(self):
assert self.distribution is None
path_to_config = os.path.join(self.managedFolder, self.configFile)
path_to_config = os.path.join(self.managed_folder, self.config_file)
try:
f = open(path_to_config, "rb")
self.distribution = pickle.load(f)
......@@ -119,7 +119,7 @@ class OSDManager(object):
return False
def __write_configuration(self):
path_to_config = os.path.join(self.managedFolder, self.configFile)
path_to_config = os.path.join(self.managed_folder, self.config_file)
f = open(path_to_config, "wb")
pickle.dump(self.distribution, f)
......@@ -426,7 +426,7 @@ class OSDManager(object):
universal_newlines=True)
folder_size = int(du.stdout.split()[0])
new_folder = folder.Folder(os.path.join(self.volumeName, self.pathOnVolume, last_2_path_elements),
new_folder = folder.Folder(os.path.join(self.volume_name, self.path_on_volume, last_2_path_elements),
folder_size,
input_folder)
if self.debug:
......@@ -460,7 +460,7 @@ class OSDManager(object):
print("Using SLURM mode for moving folders...")
if tmp_dir is None:
tmp_dir = os.path.join(self.pathToMountPoint, '.tmp_move_folder')
tmp_dir = os.path.join(self.path_to_mount_point, '.tmp_move_folder')
os.makedirs(tmp_dir, exist_ok=True)
slurm_hosts = div_util.get_slurm_hosts()
......@@ -482,7 +482,7 @@ class OSDManager(object):
host_name = osd_to_host_map[key]
command = ""
for move_folder in osd_to_folders_map[key]:
folder_path = os.path.join(self.pathToMountPoint, move_folder[len(self.volumeName) + 1:])
folder_path = os.path.join(self.path_to_mount_point, move_folder[len(self.volume_name) + 1:])
folder_tmp_path = os.path.join(tmp_dir, os.path.split(move_folder)[1])
# copy folder to temporary location
command += "srun -N1-1 --nodelist=" + host_name
......@@ -509,7 +509,7 @@ class OSDManager(object):
os.path.split(folder_id)[1])
if tmp_dir is None:
tmp_dir = os.path.join(self.pathToMountPoint, self.volumeName, '.tmp_move_folder')
tmp_dir = os.path.join(self.path_to_mount_point, self.volume_name, '.tmp_move_folder')
start_time = 0
if self.debug:
......@@ -527,11 +527,11 @@ class OSDManager(object):
if self.debug:
subprocess.run(["xtfsutil",
"--set-pattr", "1004.filenamePrefix", "--value",
"add " + folder_id + " " + new_osd_id + "", self.pathToMountPoint])
"add " + folder_id + " " + new_osd_id + "", self.path_to_mount_point])
else:
subprocess.run(["xtfsutil",
"--set-pattr", "1004.filenamePrefix", "--value",
"add " + folder_id + " " + new_osd_id + "", self.pathToMountPoint],
"add " + folder_id + " " + new_osd_id + "", self.path_to_mount_point],
stdout=subprocess.PIPE, universal_newlines=True)
# step 2: one by one, move files to tmp_location and then back to the folder, which means that they should now
......@@ -567,11 +567,11 @@ class OSDManager(object):
if self.debug:
subprocess.run(["xtfsutil",
"--set-pattr", "1004.filenamePrefix", "--value",
"remove " + folder_id + "", self.pathToMountPoint])
"remove " + folder_id + "", self.path_to_mount_point])
else:
subprocess.run(["xtfsutil",
"--set-pattr", "1004.filenamePrefix", "--value",
"remove " + folder_id + "", self.pathToMountPoint],
"remove " + folder_id + "", self.path_to_mount_point],
stdout=subprocess.PIPE, universal_newlines=True)
'''
......@@ -584,7 +584,7 @@ class OSDManager(object):
if arg_folders is not None:
for folder_for_update in arg_folders:
if not folder_for_update.startswith(self.managedFolder):
if not folder_for_update.startswith(self.managed_folder):
raise PathNotManagedException(
"The path :" + folder_for_update + "is not managed by this instance of the XtreemFS OSD"
"manager!")
......@@ -622,11 +622,11 @@ class OSDManager(object):
if not div_util.check_for_executable('xtfsutil'):
raise ExecutableNotFoundException("No xtfsutil found. Please make sure it is contained in your PATH.")
if self.osdSelectionPolicy is not "1000,1004":
if self.osd_selection_policy is not "1000,1004":
if self.debug:
subprocess.run(["xtfsutil", "--set-osp", "prefix", self.pathToMountPoint])
subprocess.run(["xtfsutil", "--set-osp", "prefix", self.path_to_mount_point])
else:
subprocess.run(["xtfsutil", "--set-osp", "prefix", self.pathToMountPoint],
subprocess.run(["xtfsutil", "--set-osp", "prefix", self.path_to_mount_point],
stdout=subprocess.PIPE, universal_newlines=True)
for new_tile in assignments:
......@@ -634,12 +634,12 @@ class OSDManager(object):
subprocess.run(["xtfsutil",
"--set-pattr", "1004.filenamePrefix", "--value",
"add " + new_tile[0] + " " + new_tile[1] + "",
self.pathToMountPoint])
self.path_to_mount_point])
else:
subprocess.run(["xtfsutil",
"--set-pattr", "1004.filenamePrefix", "--value",
"add " + new_tile[0] + " " + new_tile[1] + "",
self.pathToMountPoint],
self.path_to_mount_point],
stdout=subprocess.PIPE, universal_newlines=True)
'''
......@@ -734,7 +734,7 @@ class OSDManager(object):
if not div_util.check_for_executable('xtfsutil'):
raise ExecutableNotFoundException("No xtfsutil found. Please make sure it is contained in your PATH.")
xtfsutil = subprocess.run(["xtfsutil", self.pathToMountPoint],
xtfsutil = subprocess.run(["xtfsutil", self.path_to_mount_point],
stdout=subprocess.PIPE, universal_newlines=True)
volume_information = div_util.extract_volume_information(xtfsutil.stdout)
osd_to_ip_address = {}
......@@ -814,11 +814,11 @@ class OSDManager(object):
def get_assigned_folders(self):
assigned_folders = []
for depth_1_folder in os.listdir(self.managedFolder):
depth_1_path = os.path.join(self.managedFolder, depth_1_folder)
for depth_1_folder in os.listdir(self.managed_folder):
depth_1_path = os.path.join(self.managed_folder, depth_1_folder)
if os.path.isdir(depth_1_path):
for depth_2_folder in os.listdir(depth_1_path):
depth_2_path = os.path.join(self.managedFolder, depth_1_folder, depth_2_folder)
depth_2_path = os.path.join(self.managed_folder, depth_1_folder, depth_2_folder)
if os.path.isdir(depth_2_path):
assigned_folders.append(depth_2_path)
......@@ -829,8 +829,8 @@ class OSDManager(object):
'''
def get_target_dir(self, folder_id):
path_on_mount_point = folder_id[len(self.volumeName) + 1:]
return os.path.split(os.path.join(self.pathToMountPoint, path_on_mount_point))[0]
path_on_mount_point = folder_id[len(self.volume_name) + 1:]
return os.path.split(os.path.join(self.path_to_mount_point, path_on_mount_point))[0]
'''
remove the leading part of the path, such that only the part onto the xtreemfs volume remains, including
......@@ -839,10 +839,10 @@ class OSDManager(object):
'''
def path_on_volume(self, path):
if not path.startswith(self.managedFolder):
if not path.startswith(self.managed_folder):
raise PathNotManagedException("Path " + path + " is not managed by this instance of the XtreemFS OSD"
"manager!")
return os.path.join(self.volumeName, path[len(self.pathToMountPoint) + 1:])
return os.path.join(self.volume_name, path[len(self.path_to_mount_point) + 1:])
'''
search for the assigned folder that is a prefix of the given path on volume
......@@ -856,9 +856,9 @@ class OSDManager(object):
return None
def __str__(self):
representation = "pathToMountPoint: " + self.pathToMountPoint + " volumeName: " + self.volumeName + " pathOnVolume: " \
+ self.pathOnVolume
representation += ("\nconfigFile: " + self.configFile + "\n")
representation = "pathToMountPoint: " + self.path_to_mount_point + " volumeName: " + self.volume_name + " pathOnVolume: " \
+ self.path_on_volume
representation += ("\nconfigFile: " + self.config_file + "\n")
representation += self.distribution.description()
return representation
......
......@@ -29,10 +29,10 @@ class DataDistribution(object):
self.OSDs[new_osd.uuid] = new_osd
def add_osd_list(self, osd_list):
for osdUuid in osd_list:
if osdUuid not in self.OSDs:
new_osd = osd.OSD(osdUuid)
self.OSDs[osdUuid] = new_osd
for osd_uuid in osd_list:
if osd_uuid not in self.OSDs:
new_osd = osd.OSD(osd_uuid)
self.OSDs[osd_uuid] = new_osd
def get_osd_list(self):
osd_list = []
......
......@@ -7,7 +7,7 @@ it keeps track of the folders saved on the OSD as well as the size of the folder
class OSD(object):
def __init__(self, uuid):
self.uuid = uuid
self.totalFolderSize = 0
self.total_folder_size = 0
self.folders = {}
def add_folder(self, folder_id, folder_size):
......@@ -15,11 +15,11 @@ class OSD(object):
self.folders[folder_id] = folder_size
else:
self.folders[folder_id] += folder_size
self.totalFolderSize += folder_size
self.total_folder_size += folder_size
def remove_folder(self, folder):
if folder in self.folders.keys():
self.totalFolderSize -= self.folders[folder]
self.total_folder_size -= self.folders[folder]
del self.folders[folder]
def update_folder(self, folder_id, size):
......@@ -32,5 +32,5 @@ class OSD(object):
def __str__(self):
return "osd: '" + self.uuid \
+ "' totalFolderSize: " + str(self.totalFolderSize) \
+ "' totalFolderSize: " + str(self.total_folder_size) \
+ " number of folders: " + str(len(self.folders))
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment