das.py 4.68 KB
Newer Older
1
2
3
4
import argparse

import sys

Felix Seibert's avatar
Felix Seibert committed
5
6
7
8
#import .xtreemfs_client.OSDManager as OSDManager
#import .xtreemfs_client.verify as verify

#from .OSDManager import OSDManager
Felix Seibert's avatar
Felix Seibert committed
9
from xtreemfs_client import OSDManager
Felix Seibert's avatar
Felix Seibert committed
10
#from .xtreemfs_client import verify
Felix Seibert's avatar
Felix Seibert committed
11
from xtreemfs_client import verify
12
13

"""
14
das - data add script. basically a command line wrapper for OSDManager.
15
16
17
18
19
20
21
22
23
24

please use absolute paths only!
"""

parser = argparse.ArgumentParser(description="Copy and manage files in xtreemfs volumes")

parser.add_argument("--copy", "-c", action='store_const', const=True, default=False,
                    help='Copy from the directories (comma-separated) specified by --source-folders to the'
                         ' folder specified by target_folder, which must be onto an XtreemFS volume')

Felix Seibert's avatar
Felix Seibert committed
25
parser.add_argument("--dont-execute-copy", action='store_const', const=True, default=False)
26

27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
parser.add_argument("--new-folders", action='store_const', const=True, default=False,
                    help='For each subdirectory of each folder specified by  --source-folders, create'
                         ' a corresponding folder in the folder specified by target_folder and'
                         ' assign an OSD to this directory. You should call --update with the same'
                         ' --source-folders and target_folder arguments after copying data'
                         ' manually, e.g., without this tool.')

parser.add_argument("--update", "-u", action='store_const', const=True, default=False,
                    help='Update the data layout such that it matches the physical layout. This is'
                         ' necessary after copying manually, e.g., without this tool.')

parser.add_argument("--source-folders", "-s", nargs=1)
parser.add_argument("target-folder", nargs=1)

parser.add_argument("--remote-source", "-r", nargs=1,
                    help='Use a remote source to copy from. '
                         'The remote host must be reachable via ssh without password.')
parser.add_argument("--random-layout", "-x", action='store_const', const=True, default=False)
45
parser.add_argument("--random-osd-assignment", action='store_const', const=True, default=False)
46
47
48
49
50
51
52
53
54
parser.add_argument("--environment", "-e", choices=['LOCAL', 'SLURM', 'HU_CLUSTER'], default='LOCAL')

parser.add_argument("--print", "-p", action='store_const', const=True, default=False,
                    help='Print the layout and exit. You need to specify a target folder.')

parser.add_argument("--debug", "-d", action='store_const', const=True, default=False)

parser.add_argument("--verify", "-v", action='store_const', const=True, default=False)

55
56
57
58
59
parser.add_argument("--create-from-existing-files", action='store_const',const=True, default=False,
                    help='creates a data distribution based on the files already present,'
                         'and changes the physical location of files to match the locations prescribed by the '
                         'distribution.')

60
61
62
63
64
parser.add_argument("--fix-internally", action='store_const', const=True, default=False,
                    help='indicate whether xtreemfs internal functions should be used to fix the physical'
                         'layout. otherwise files will be temporarily located outside xtreemfs,'
                         'increasing the chance for data loss.')

65
66
67
68
69
70
71
72
73
args = parser.parse_args()

if args.debug:
    print("args: ")
    print(args)

if args.verify:
    good_layout = verify.verify_gms_folder(vars(args)['target-folder'][0])
    print("good_layout: ", good_layout)
74
75
    # if not good_layout:
    #     verify.print_tree(vars(args)['target-folder'][0])
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
    sys.exit(0)

x_man = OSDManager.OSDManager(vars(args)['target-folder'][0], debug=args.debug)

if args.print:
    print(x_man)

if args.copy:
    if len(args.source_folders) == 0:
        print("You must specify folders to copy with --source-folders")
        sys.exit(1)

    else:
        folders = args.source_folders[0].split(',')
        x_man.copy_folders(folders, environment=args.environment, apply_layout=(not args.random_layout),
91
92
                           remote_source=args.remote_source[0], random_osd_assignment=args.random_osd_assignment,
                           execute_copy=(not args.dont_execute_copy))
93
94
95
96
97
98
99
100
101
102
103
104
105

elif args.new_folders:
    if len(args.source_folders) == 0:
        print("You must specify folders to create with --source-folders")
        sys.exit(1)

    else:
        folders = args.source_folders[0].split(',')
        x_man.create_empty_folders(folders)

elif args.update:
    x_man.update()
    print(x_man)
106
107

elif args.create_from_existing_files:
108
109
    x_man.create_distribution_from_existing_files(fix_layout_internally=args.fix_internally,
                                                  environment=args.environment)