Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Felix Seibert
xtreemfs_client
Commits
a488b8c0
Commit
a488b8c0
authored
Dec 01, 2018
by
Felix Seibert
Browse files
add option to rebalance an existing osd to folder assignment
parent
e86e345d
Changes
2
Hide whitespace changes
Inline
Side-by-side
xtreemfs_client/OSDManager.py
View file @
a488b8c0
...
...
@@ -201,6 +201,69 @@ class OSDManager(object):
total_time
=
round
(
time
.
time
()
-
start_time
)
print
(
"fixed physical layout of existing files in secs: "
+
str
(
total_time
))
def
rebalance_existing_assignment
(
self
,
rebalance_algorithm
=
'lpt'
,
fix_layout_internally
=
True
,
max_files_in_progress
=
10000
,
environment
=
'LOCAL'
,
movement_strategy
=
'osd_balanced'
):
if
self
.
debug
:
print
(
"rebalancing existing distribution... osd manager:
\n
"
+
str
(
self
))
start_time
=
time
.
time
()
self
.
update
()
update_time
=
round
(
time
.
time
()
-
start_time
)
if
self
.
debug
:
print
(
"updated folder sizes in secs: "
+
str
(
update_time
))
start_time
=
time
.
time
()
if
rebalance_algorithm
is
'rebalance_one'
:
movements
=
self
.
distribution
.
rebalance_one_folder
()
elif
rebalance_algorithm
is
'two_step_opt'
:
movements
=
self
.
distribution
.
rebalance_two_steps_optimal_matching
()
elif
rebalance_algorithm
is
'two_step_rnd'
:
movements
=
self
.
distribution
.
rebalance_two_steps_random_matching
()
else
:
movements
=
self
.
distribution
.
rebalance_lpt
()
if
self
.
debug
:
rebalance_time
=
round
(
time
.
time
()
-
start_time
)
print
(
"rebalanced assignment in secs: "
+
str
(
rebalance_time
))
start_time
=
time
.
time
()
if
fix_layout_internally
:
# TODO use movements to make realize_placement more efficient
# TODO (for the first calculation of files that need to be moved)
placement_realizer
=
\
physicalPlacementRealizer
.
PhysicalPlacementRealizer
(
self
,
debug
=
self
.
debug
,
max_files_in_progress
=
max_files_in_progress
)
placement_realizer
.
realize_placement
(
strategy
=
movement_strategy
)
elif
environment
==
'SLURM'
:
target_balanced
=
1
# 0 is origin balanced, 1 is target balanced
# we use self.__generate_move_commands_slurm() to do so.
# this either ignores the origin or the target information.
# TODO can we do this more intelligently?
osd_to_folders_map
=
{}
for
folder_to_move
in
movements
.
keys
():
# we have a choice: origin or target balanced.
osd_id
=
movements
[
folder_to_move
][
target_balanced
]
if
osd_id
in
osd_to_folders_map
:
osd_to_folders_map
[
osd_id
].
append
(
folder_to_move
)
else
:
osd_to_folders_map
[
osd_id
]
=
[
folder_to_move
]
move_commands
=
self
.
__generate_move_commands_slurm
(
osd_to_folders_map
)
self
.
__execute_commands
(
move_commands
)
else
:
self
.
fix_physical_layout_externally
()
if
self
.
debug
:
total_time
=
round
(
time
.
time
()
-
start_time
)
print
(
"fixed physical layout of existing files in secs: "
+
str
(
total_time
))
def
fix_physical_layout_externally
(
self
):
"""
fixes the physical layout, such that it matches the data distribution described in self.distribution.
...
...
xtreemfs_client/das.py
View file @
a488b8c0
...
...
@@ -52,6 +52,9 @@ parser.add_argument("--create-from-existing-files", action='store_const', const=
'and changes the physical location of files to match the locations prescribed by the '
'distribution.'
)
parser
.
add_argument
(
"--rebalance-existing-assignment"
,
action
=
'store_const'
,
const
=
True
,
default
=
False
,
help
=
'rebalances an existing osd to folder assignment, using the lpt rebalancing method.'
)
parser
.
add_argument
(
"--fix-internally"
,
action
=
'store_const'
,
const
=
True
,
default
=
False
,
help
=
'indicate whether xtreemfs internal functions should be used to fix the physical'
'layout. otherwise files will be temporarily located outside xtreemfs,'
...
...
@@ -107,3 +110,9 @@ elif args.create_from_existing_files:
environment
=
args
.
environment
,
max_files_in_progress
=
int
(
args
.
max_files_in_progress
[
0
]),
movement_strategy
=
args
.
movement_strategy
[
0
])
elif
args
.
rebalance_existing_assignment
:
x_man
.
rebalance_existing_assignment
(
fix_layout_internally
=
args
.
fix_internally
,
environment
=
args
.
environment
,
max_files_in_progress
=
int
(
args
.
max_files_in_progress
[
0
]),
movement_strategy
=
args
.
movement_strategy
[
0
])
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment