Commit a4a5f8fb authored by Felix Seibert's avatar Felix Seibert
Browse files

implementing another way for fixing the physical data layout

parent db4ea75e
......@@ -130,7 +130,7 @@ class OSDManager(object):
using XtreemFS' read-only replication strategy.
'''
def create_distribution_from_existing_files(self, apply_layout=True):
def create_distribution_from_existing_files(self, fix_layout_internally=True, apply_layout=True):
if self.debug:
print("creating distribution from existing files. osd manager: " + str(self))
......@@ -163,7 +163,27 @@ class OSDManager(object):
print("osd manager after new folders have been added to data distribution:")
print(str(self))
self.fix_physical_layout()
if fix_layout_internally:
self.fix_physical_layout_internally()
else:
self.fix_physical_layout_externally()
'''
fixes the physical layout, such that it matches the data distribution described in self.distribution.
this is realized by calling move_folder_to_osd on all folders managed by this distribution.
'''
def fix_physical_layout_externally(self):
start_time = time.time()
managed_folders = self.get_assigned_folders()
for managed_folder in managed_folders:
folder_id = self.path_on_volume(managed_folder)
osd_for_folder = self.distribution.get_containing_osd(folder_id)
self.move_folder_to_osd(folder_id, osd_for_folder.uuid)
total_time = time.time() - start_time
if self.debug:
print("externally fixed physical data layout in secs: " + str(round(total_time)))
'''
fixes the physical layout, such that it matches the data distribution described in self.distribution
......@@ -181,7 +201,12 @@ class OSDManager(object):
'''
def fix_physical_layout(self, repeat_delete_interval_secs=15, iteration=0):
def fix_physical_layout_internally(self, repeat_delete_interval_secs=15, iteration=0, start_time=None):
if start_time is None:
fix_start_time = time.time()
else:
fix_start_time = start_time
if self.debug:
print("starting to fix physical layout...this is fix-iteration " + str(iteration))
# list 1
......@@ -227,7 +252,9 @@ class OSDManager(object):
# end of recursion condition
if len(create_replica_command_list) == 0 and len(delete_replica_command_list) == 0:
if self.debug:
print("fixing physical layout done after " + str(iteration) + " fix-iterations.")
fix_total_time = round(time.time() - fix_start_time)
print("internally fixed physical layout in " + str(iteration) + " fix-iterations, "
"took secs: " + str(fix_total_time))
return
# run commands
......@@ -307,7 +334,7 @@ class OSDManager(object):
end_time = time.time()
print("deleting replicas done in in " + str(round(end_time - start_time)) + " sec.")
self.fix_physical_layout(iteration=iteration + 1)
self.fix_physical_layout_internally(iteration=iteration + 1, start_time=fix_start_time)
'''
create empty folders and assign OSDs.
......@@ -419,7 +446,14 @@ class OSDManager(object):
folder_path = os.path.join(self.get_target_dir(folder_id),
os.path.split(folder_id)[1])
os.makedirs(tmp_dir)
start_time = 0
if self.debug:
start_time = time.time()
if self.debug:
print("externally moving folder " + folder_id + " to osd: " + new_osd_id)
os.makedirs(tmp_dir, exist_ok=True)
if not div_util.check_for_executable('xtfsutil'):
raise ExecutableNotFoundException("No xtfsutil found. Please make sure it is contained in your PATH.")
......@@ -447,6 +481,11 @@ class OSDManager(object):
shutil.rmtree(tmp_dir, ignore_errors=True)
if self.debug:
total_time = time.time() - start_time
print("externally moved folder " + folder_id +
" to osd: " + new_osd_id + " in secs: " + str(round(total_time)))
'''
removes a folder from the distribution. this does NOT delete the folder from the file system.
'''
......
......@@ -57,6 +57,11 @@ parser.add_argument("--create-from-existing-files", action='store_const',const=T
'and changes the physical location of files to match the locations prescribed by the '
'distribution.')
parser.add_argument("--fix-internally", action='store_const', const=True, default=False,
help='indicate whether xtreemfs internal functions should be used to fix the physical'
'layout. otherwise files will be temporarily located outside xtreemfs,'
'increasing the chance for data loss.')
args = parser.parse_args()
if args.debug:
......@@ -100,4 +105,4 @@ elif args.update:
print(x_man)
elif args.create_from_existing_files:
x_man.create_distribution_from_existing_files()
x_man.create_distribution_from_existing_files(fix_layout_internally=args.fix_internally)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment