das.py 5.48 KB
Newer Older
1
2
3
4
import argparse

import sys

Felix Seibert's avatar
Felix Seibert committed
5
6
from xtreemfs_client import OSDManager
from xtreemfs_client import verify
7
8

"""
9
das - data add script. basically a command line wrapper for OSDManager.
10
11
12
13
14
15
16
17
18
19

please use absolute paths only!
"""

parser = argparse.ArgumentParser(description="Copy and manage files in xtreemfs volumes")

parser.add_argument("--copy", "-c", action='store_const', const=True, default=False,
                    help='Copy from the directories (comma-separated) specified by --source-folders to the'
                         ' folder specified by target_folder, which must be onto an XtreemFS volume')

Felix Seibert's avatar
Felix Seibert committed
20
parser.add_argument("--dont-execute-copy", action='store_const', const=True, default=False)
21

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
parser.add_argument("--new-folders", action='store_const', const=True, default=False,
                    help='For each subdirectory of each folder specified by  --source-folders, create'
                         ' a corresponding folder in the folder specified by target_folder and'
                         ' assign an OSD to this directory. You should call --update with the same'
                         ' --source-folders and target_folder arguments after copying data'
                         ' manually, e.g., without this tool.')

parser.add_argument("--update", "-u", action='store_const', const=True, default=False,
                    help='Update the data layout such that it matches the physical layout. This is'
                         ' necessary after copying manually, e.g., without this tool.')

parser.add_argument("--source-folders", "-s", nargs=1)
parser.add_argument("target-folder", nargs=1)

parser.add_argument("--remote-source", "-r", nargs=1,
                    help='Use a remote source to copy from. '
                         'The remote host must be reachable via ssh without password.')
parser.add_argument("--random-layout", "-x", action='store_const', const=True, default=False)
40
parser.add_argument("--random-osd-assignment", action='store_const', const=True, default=False)
41
parser.add_argument("--random-seed", nargs=1)
42
43
44
45
46
47
48
49
50
parser.add_argument("--environment", "-e", choices=['LOCAL', 'SLURM', 'HU_CLUSTER'], default='LOCAL')

parser.add_argument("--print", "-p", action='store_const', const=True, default=False,
                    help='Print the layout and exit. You need to specify a target folder.')

parser.add_argument("--debug", "-d", action='store_const', const=True, default=False)

parser.add_argument("--verify", "-v", action='store_const', const=True, default=False)

51
parser.add_argument("--create-from-existing-files", action='store_const', const=True, default=False,
52
53
54
55
                    help='creates a data distribution based on the files already present,'
                         'and changes the physical location of files to match the locations prescribed by the '
                         'distribution.')

56
57
58
parser.add_argument("--rebalance-existing-assignment", action='store_const', const=True, default=False,
                    help='rebalances an existing osd to folder assignment, using the lpt rebalancing method.')

59
60
61
62
63
parser.add_argument("--fix-internally", action='store_const', const=True, default=False,
                    help='indicate whether xtreemfs internal functions should be used to fix the physical'
                         'layout. otherwise files will be temporarily located outside xtreemfs,'
                         'increasing the chance for data loss.')

64
parser.add_argument("--max-files-in-progress", nargs=1)
65
parser.add_argument("--movement-strategy", nargs=1)
66

67
68
69
70
71
72
73
74
75
args = parser.parse_args()

if args.debug:
    print("args: ")
    print(args)

if args.verify:
    good_layout = verify.verify_gms_folder(vars(args)['target-folder'][0])
    print("good_layout: ", good_layout)
76
77
    # if not good_layout:
    #     verify.print_tree(vars(args)['target-folder'][0])
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
    sys.exit(0)

x_man = OSDManager.OSDManager(vars(args)['target-folder'][0], debug=args.debug)

if args.print:
    print(x_man)

if args.copy:
    if len(args.source_folders) == 0:
        print("You must specify folders to copy with --source-folders")
        sys.exit(1)

    else:
        folders = args.source_folders[0].split(',')
        x_man.copy_folders(folders, environment=args.environment, apply_layout=(not args.random_layout),
93
                           remote_source=args.remote_source[0], random_osd_assignment=args.random_osd_assignment,
94
                           random_seed=args.random_seed[0], execute_copy=(not args.dont_execute_copy))
95
96
97
98
99
100
101
102
103
104
105
106
107

elif args.new_folders:
    if len(args.source_folders) == 0:
        print("You must specify folders to create with --source-folders")
        sys.exit(1)

    else:
        folders = args.source_folders[0].split(',')
        x_man.create_empty_folders(folders)

elif args.update:
    x_man.update()
    print(x_man)
108
109

elif args.create_from_existing_files:
110
    x_man.create_distribution_from_existing_files(fix_layout_internally=args.fix_internally,
111
                                                  environment=args.environment,
112
113
                                                  max_files_in_progress=int(args.max_files_in_progress[0]),
                                                  movement_strategy=args.movement_strategy[0])
114
115
116
117
118
119

elif args.rebalance_existing_assignment:
    x_man.rebalance_existing_assignment(fix_layout_internally=args.fix_internally,
                                        environment=args.environment,
                                        max_files_in_progress=int(args.max_files_in_progress[0]),
                                        movement_strategy=args.movement_strategy[0])