Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Felix Seibert
xtreemfs_client
Commits
bf81b68b
Commit
bf81b68b
authored
Mar 26, 2018
by
Felix Seibert
Browse files
implementing slurm environment move method
parent
6fb5ce63
Changes
1
Hide whitespace changes
Inline
Side-by-side
xtreemfs_client/OSDManager.py
View file @
bf81b68b
...
...
@@ -130,7 +130,8 @@ class OSDManager(object):
using XtreemFS' read-only replication strategy.
'''
def
create_distribution_from_existing_files
(
self
,
fix_layout_internally
=
True
,
apply_layout
=
True
):
def
create_distribution_from_existing_files
(
self
,
fix_layout_internally
=
True
,
apply_layout
=
True
,
environment
=
'LOCAL'
,):
if
self
.
debug
:
print
(
"creating distribution from existing files. osd manager: "
+
str
(
self
))
...
...
@@ -166,7 +167,22 @@ class OSDManager(object):
if
fix_layout_internally
:
self
.
fix_physical_layout_internally
()
else
:
self
.
fix_physical_layout_externally
()
if
environment
is
'SLURM'
:
osd_list
=
self
.
distribution
.
get_osd_list
()
osd_to_folders_map
=
{}
all_folders
=
[]
for
osd
in
osd_list
:
osd_to_folders_map
[
osd
]
=
[]
for
osds_folder
in
osd
.
folders
:
all_folders
.
append
(
osds_folder
)
for
input_folder
in
all_folders
:
osd_for_tile
=
self
.
distribution
.
get_containing_osd
(
input_folder
.
id
).
uuid
osd_to_folders_map
[
osd_for_tile
].
append
(
input_folder
)
move_commands
=
self
.
__generate_move_commands_slurm
(
osd_to_folders_map
)
self
.
__execute_copy_commands
(
move_commands
)
else
:
self
.
fix_physical_layout_externally
()
'''
fixes the physical layout, such that it matches the data distribution described in self.distribution.
...
...
@@ -467,13 +483,17 @@ class OSDManager(object):
for
move_folder
in
osd_to_folders_map
[
key
]:
folder_path
=
os
.
path
.
join
(
self
.
pathToMountPoint
,
move_folder
.
folder_id
)
folder_tmp_path
=
os
.
path
.
join
(
tmp_dir
,
move_folder
.
folder_id
)
# copy folder to temporary location
command
+=
"srun -N1-1 --nodelist="
+
host_name
command
+=
" move "
+
folder_path
command
+=
" "
+
tmp_dir
command
+=
" ;"
command
+=
" move "
+
folder_tmp_path
command
+=
" "
+
folder_path
+
" ; "
command
+=
" cp -r "
+
folder_path
+
" "
+
tmp_dir
+
" ; "
# delete folder within the managed xtreemfs folder
command
+=
"srun -N1-1 --nodelist="
+
host_name
command
+=
" rm -r "
+
folder_path
+
" ; "
# copy folder back from temporary location
command
+=
" cp -r "
+
folder_tmp_path
+
" "
+
os
.
path
.
split
(
folder_path
)[
0
]
+
" ; "
# delete folder from temporary location
command
+=
"srun -N1-1 --nodelist="
+
host_name
command
+=
" rm -r "
+
folder_tmp_path
+
" ; "
if
len
(
osd_to_folders_map
[
key
])
>
0
:
command_list
.
append
(
command
)
# command_list.append(command)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment