Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Felix Seibert
xtreemfs_client
Commits
c1ceb491
Commit
c1ceb491
authored
Mar 29, 2018
by
Felix Seibert
Browse files
refactoring: differentiate between os folders and assigned folders
parent
fef86bab
Changes
1
Hide whitespace changes
Inline
Side-by-side
xtreemfs_client/OSDManager.py
View file @
c1ceb491
...
...
@@ -138,8 +138,8 @@ class OSDManager(object):
if
not
div_util
.
check_for_executable
(
'du'
):
raise
ExecutableNotFoundException
(
"No du found. Please make sure it is contained in your PATH."
)
existing_folders
=
self
.
get_
assigned_folder
s
()
managed
_folders
=
[]
existing_folders
=
self
.
get_
depth_2_subdirectorie
s
()
new
_folders
=
[]
for
one_folder
in
existing_folders
:
du
=
subprocess
.
run
([
"du"
,
"-s"
,
one_folder
],
stdout
=
subprocess
.
PIPE
,
universal_newlines
=
True
)
...
...
@@ -149,9 +149,9 @@ class OSDManager(object):
new_folder
=
folder
.
Folder
(
self
.
path_on_volume
(
one_folder
),
folder_size
,
None
)
managed
_folders
.
append
(
new_folder
)
new
_folders
.
append
(
new_folder
)
new_assignments
=
self
.
distribution
.
add_folders
(
managed
_folders
)
new_assignments
=
self
.
distribution
.
add_folders
(
new
_folders
)
if
apply_layout
:
self
.
apply_osd_assignments
(
new_assignments
)
...
...
@@ -201,9 +201,8 @@ class OSDManager(object):
"""
if
self
.
debug
:
print
(
"fixing physical layout externally..."
)
managed_folders
=
self
.
get_assigned_folders
()
for
managed_folder
in
managed_folders
:
folder_id
=
self
.
path_on_volume
(
managed_folder
)
managed_folders
=
self
.
get_assigned_folder_ids
()
for
folder_id
in
managed_folders
:
osd_for_folder
=
self
.
distribution
.
get_containing_osd
(
folder_id
)
self
.
move_folder_to_osd
(
folder_id
,
osd_for_folder
.
uuid
)
...
...
@@ -232,7 +231,7 @@ class OSDManager(object):
delete_replica_command_list
=
[]
# create commands
managed_folders
=
self
.
get_
assigned_folder
s
()
managed_folders
=
self
.
get_
depth_2_subdirectorie
s
()
for
managed_folder
in
managed_folders
:
for
directory
in
os
.
walk
(
managed_folder
):
for
filename
in
directory
[
2
]:
...
...
@@ -419,6 +418,7 @@ class OSDManager(object):
universal_newlines
=
True
)
folder_size
=
int
(
du
.
stdout
.
split
()[
0
])
# as the folder_id is generated from the copy source, we cannot call get_path_on_volume to get the foler_id
new_folder
=
folder
.
Folder
(
os
.
path
.
join
(
self
.
volume_name
,
self
.
path_on_volume
,
last_2_path_elements
),
folder_size
,
input_folder
)
...
...
@@ -475,7 +475,7 @@ class OSDManager(object):
host_name
=
osd_to_host_map
[
key
]
command
=
""
for
move_folder
in
osd_to_folders_map
[
key
]:
folder_path
=
os
.
path
.
join
(
self
.
path_to_mount_point
,
move_folder
[
len
(
self
.
volume_name
)
+
1
:]
)
folder_path
=
self
.
get_absolute_file_path
(
move_folder
)
folder_tmp_path
=
os
.
path
.
join
(
tmp_dir
,
os
.
path
.
split
(
move_folder
)[
1
])
# copy folder to temporary location
command
+=
"srun -N1-1 --nodelist="
+
host_name
...
...
@@ -501,7 +501,7 @@ class OSDManager(object):
os
.
path
.
split
(
folder_id
)[
1
])
if
tmp_dir
is
None
:
tmp_dir
=
os
.
path
.
join
(
self
.
path_to_mount_point
,
self
.
volume_name
,
'.tmp_move_folder'
)
tmp_dir
=
os
.
path
.
join
(
self
.
path_to_mount_point
,
'.tmp_move_folder'
)
start_time
=
0
if
self
.
debug
:
...
...
@@ -582,7 +582,7 @@ class OSDManager(object):
folders
=
arg_folders
if
arg_folders
is
None
:
folders
=
self
.
get_
assigned_folder
s
()
folders
=
self
.
get_
depth_2_subdirectorie
s
()
for
folder_for_update
in
folders
:
folder_id
=
self
.
path_on_volume
(
folder_for_update
)
...
...
@@ -789,41 +789,53 @@ class OSDManager(object):
print
(
str
(
terminated_process
.
communicate
()))
print
(
"Executing commands done."
)
def
get_
assigned_folder
s
(
self
):
def
get_
depth_2_subdirectorie
s
(
self
):
"""
creates a list of all folders, managed by this XtreemFS OSD manager, that have an assigned OSD
TODO this actually just returns a list of all level 2 subdirectories, independently of whether they are actually
assigned to some OSD or not. i think that this is not the expected behaviour.
creates a list of all depth 2 subdirectories of self.managed_folder
"""
assigned_folder
s
=
[]
subdirectorie
s
=
[]
for
depth_1_folder
in
os
.
listdir
(
self
.
managed_folder
):
depth_1_path
=
os
.
path
.
join
(
self
.
managed_folder
,
depth_1_folder
)
if
os
.
path
.
isdir
(
depth_1_path
):
for
depth_2_folder
in
os
.
listdir
(
depth_1_path
):
depth_2_path
=
os
.
path
.
join
(
self
.
managed_folder
,
depth_1_folder
,
depth_2_folder
)
if
os
.
path
.
isdir
(
depth_2_path
):
assigned_folders
.
append
(
depth_2_path
)
subdirectories
.
append
(
depth_2_path
)
return
subdirectories
def
get_assigned_folder_ids
(
self
):
"""
creates a list of ids of all assigned folders (folders assigned to OSDs)
"""
osd_list
=
self
.
distribution
.
get_osd_list
()
assigned_folders
=
[]
for
osd
in
osd_list
:
for
one_folder
in
self
.
distribution
.
OSDs
[
osd
].
folders
:
assigned_folders
.
append
(
one_folder
)
return
assigned_folders
def
get_target_dir
(
self
,
folder_id
):
"""
gets the path to the target dir, given the folder_id
gets the path to the target dir
(where to copy the folder)
, given the folder_id
"""
path_on_mount_point
=
folder_id
[
len
(
self
.
volume_name
)
+
1
:]
return
os
.
path
.
split
(
os
.
path
.
join
(
self
.
path_to_mount_point
,
path_on_mount_point
))[
0
]
return
os
.
path
.
split
(
self
.
get_absolute_file_path
(
folder_id
))[
0
]
def
path_on_volume
(
self
,
path
):
def
get_
path_on_volume
(
self
,
path
):
"""
remove the leading part of the path, such that only the part onto the xtreemfs volume remains, including
the volume itself.
throws an exception when the path is not managed by this XtreemFS OSD manager.
use this method to calculate the folder_id.
"""
if
not
path
.
startswith
(
self
.
managed_folder
):
raise
PathNotManagedException
(
"Path "
+
path
+
" is not managed by this instance of the XtreemFS OSD"
"manager!"
)
return
os
.
path
.
join
(
self
.
volume_name
,
path
[
len
(
self
.
path_to_mount_point
)
+
1
:])
def
get_absolute_file_path
(
self
,
folder_id
):
return
os
.
path
.
join
(
self
.
path_to_mount_point
,
folder_id
[
len
(
self
.
volume_name
)
+
1
:])
def
get_containing_folder_id
(
self
,
path_on_volume
):
"""
search for the assigned folder that is a prefix of the given path on volume
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment