Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Felix Seibert
xtreemfs_client
Commits
93f2172b
Commit
93f2172b
authored
Dec 03, 2018
by
Felix Seibert
Browse files
optional random seed specification for random assignment
parent
5db4700d
Changes
3
Hide whitespace changes
Inline
Side-by-side
xtreemfs_client/OSDManager.py
View file @
93f2172b
...
@@ -307,7 +307,7 @@ class OSDManager(object):
...
@@ -307,7 +307,7 @@ class OSDManager(object):
self
.
__write_configuration
()
self
.
__write_configuration
()
def
copy_folders
(
self
,
folders
,
environment
=
'LOCAL'
,
remote_source
=
None
,
sshfs_mount_dir
=
'/tmp/sshfs_tmp_mnt'
,
def
copy_folders
(
self
,
folders
,
environment
=
'LOCAL'
,
remote_source
=
None
,
sshfs_mount_dir
=
'/tmp/sshfs_tmp_mnt'
,
apply_layout
=
True
,
execute_copy
=
True
,
random_osd_assignment
=
False
):
apply_layout
=
True
,
execute_copy
=
True
,
random_osd_assignment
=
False
,
random_seed
=
None
):
"""
"""
copy a list of given folders into the managed folder, assigning OSDs to new folders and updating
copy a list of given folders into the managed folder, assigning OSDs to new folders and updating
self.dataDistribution
self.dataDistribution
...
@@ -369,7 +369,8 @@ class OSDManager(object):
...
@@ -369,7 +369,8 @@ class OSDManager(object):
if
self
.
debug
:
if
self
.
debug
:
print
(
"OSDManager: random_osd_assignment: "
+
str
(
random_osd_assignment
))
print
(
"OSDManager: random_osd_assignment: "
+
str
(
random_osd_assignment
))
new_assignments
=
self
.
distribution
.
add_folders
(
new_folders
,
random_osd_assignment
=
random_osd_assignment
)
new_assignments
=
self
.
distribution
.
add_folders
(
new_folders
,
random_osd_assignment
=
random_osd_assignment
,
random_seed
=
random_seed
)
if
apply_layout
:
if
apply_layout
:
self
.
apply_osd_assignments
(
new_assignments
)
self
.
apply_osd_assignments
(
new_assignments
)
elif
self
.
debug
:
elif
self
.
debug
:
...
...
xtreemfs_client/das.py
View file @
93f2172b
...
@@ -38,6 +38,7 @@ parser.add_argument("--remote-source", "-r", nargs=1,
...
@@ -38,6 +38,7 @@ parser.add_argument("--remote-source", "-r", nargs=1,
'The remote host must be reachable via ssh without password.'
)
'The remote host must be reachable via ssh without password.'
)
parser
.
add_argument
(
"--random-layout"
,
"-x"
,
action
=
'store_const'
,
const
=
True
,
default
=
False
)
parser
.
add_argument
(
"--random-layout"
,
"-x"
,
action
=
'store_const'
,
const
=
True
,
default
=
False
)
parser
.
add_argument
(
"--random-osd-assignment"
,
action
=
'store_const'
,
const
=
True
,
default
=
False
)
parser
.
add_argument
(
"--random-osd-assignment"
,
action
=
'store_const'
,
const
=
True
,
default
=
False
)
parser
.
add_argument
(
"--random-seed"
,
nargs
=
1
)
parser
.
add_argument
(
"--environment"
,
"-e"
,
choices
=
[
'LOCAL'
,
'SLURM'
,
'HU_CLUSTER'
],
default
=
'LOCAL'
)
parser
.
add_argument
(
"--environment"
,
"-e"
,
choices
=
[
'LOCAL'
,
'SLURM'
,
'HU_CLUSTER'
],
default
=
'LOCAL'
)
parser
.
add_argument
(
"--print"
,
"-p"
,
action
=
'store_const'
,
const
=
True
,
default
=
False
,
parser
.
add_argument
(
"--print"
,
"-p"
,
action
=
'store_const'
,
const
=
True
,
default
=
False
,
...
@@ -90,7 +91,7 @@ if args.copy:
...
@@ -90,7 +91,7 @@ if args.copy:
folders
=
args
.
source_folders
[
0
].
split
(
','
)
folders
=
args
.
source_folders
[
0
].
split
(
','
)
x_man
.
copy_folders
(
folders
,
environment
=
args
.
environment
,
apply_layout
=
(
not
args
.
random_layout
),
x_man
.
copy_folders
(
folders
,
environment
=
args
.
environment
,
apply_layout
=
(
not
args
.
random_layout
),
remote_source
=
args
.
remote_source
[
0
],
random_osd_assignment
=
args
.
random_osd_assignment
,
remote_source
=
args
.
remote_source
[
0
],
random_osd_assignment
=
args
.
random_osd_assignment
,
execute_copy
=
(
not
args
.
dont_execute_copy
))
random_seed
=
args
.
random_seed
[
0
],
execute_copy
=
(
not
args
.
dont_execute_copy
))
elif
args
.
new_folders
:
elif
args
.
new_folders
:
if
len
(
args
.
source_folders
)
==
0
:
if
len
(
args
.
source_folders
)
==
0
:
...
...
xtreemfs_client/dataDistribution.py
View file @
93f2172b
...
@@ -208,7 +208,8 @@ class DataDistribution(object):
...
@@ -208,7 +208,8 @@ class DataDistribution(object):
ignore_osd_capacities
=
True
,
ignore_osd_capacities
=
True
,
random_osd_assignment
=
False
,
random_osd_assignment
=
False
,
ignore_folder_sizes
=
False
,
ignore_folder_sizes
=
False
,
debug
=
False
):
debug
=
False
,
random_seed
=
None
):
"""
"""
adds a list of folders to the data distribution.
adds a list of folders to the data distribution.
if not specified otherwise, the assignments are calculated using the LPT algorithm.
if not specified otherwise, the assignments are calculated using the LPT algorithm.
...
@@ -242,6 +243,9 @@ class DataDistribution(object):
...
@@ -242,6 +243,9 @@ class DataDistribution(object):
# this information must be returned
# this information must be returned
osds_for_new_folders
=
[]
osds_for_new_folders
=
[]
if
random_osd_assignment
:
random
.
seed
(
random_seed
)
# totally random OSD assignment, ignoring OSD capacities
# totally random OSD assignment, ignoring OSD capacities
# (might lead to I/O errors when too many groups are assigned to an OSD)
# (might lead to I/O errors when too many groups are assigned to an OSD)
if
random_osd_assignment
and
ignore_osd_capacities
and
not
ignore_folder_sizes
:
if
random_osd_assignment
and
ignore_osd_capacities
and
not
ignore_folder_sizes
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment