|
1
|
|
|
from __future__ import print_function |
|
2
|
|
|
|
|
3
|
|
|
import re |
|
4
|
|
|
import fcntl |
|
5
|
|
|
import binascii |
|
6
|
|
|
import logging |
|
7
|
|
|
import hashlib |
|
8
|
|
|
import unicodedata |
|
9
|
|
|
import json |
|
10
|
|
|
|
|
11
|
|
|
from distutils.util import strtobool |
|
12
|
|
|
from subprocess import Popen, PIPE |
|
13
|
|
|
from contextlib import contextmanager |
|
14
|
|
|
|
|
15
|
|
|
|
|
16
|
|
|
def print_boxed(string): |
|
17
|
|
|
splitted_string = string.split('\n') |
|
18
|
|
|
max_len = max(map(len, splitted_string)) |
|
19
|
|
|
box_line = u"\u2500" * (max_len + 2) |
|
20
|
|
|
|
|
21
|
|
|
out = u"\u250c" + box_line + u"\u2510\n" |
|
22
|
|
|
out += '\n'.join([u"\u2502 {} \u2502".format(line.ljust(max_len)) for line in splitted_string]) |
|
23
|
|
|
out += u"\n\u2514" + box_line + u"\u2518" |
|
24
|
|
|
print(out) |
|
25
|
|
|
|
|
26
|
|
|
|
|
27
|
|
|
def yes_no_prompt(query, default=None): |
|
28
|
|
|
available_prompts = {None: " [y/n] ", 'y': " [Y/n] ", 'n': " [y/N] "} |
|
29
|
|
|
|
|
30
|
|
|
if default not in available_prompts: |
|
31
|
|
|
raise ValueError("Invalid default: '{}'".format(default)) |
|
32
|
|
|
|
|
33
|
|
|
while True: |
|
34
|
|
|
try: |
|
35
|
|
|
answer = raw_input("{0}{1}".format(query, available_prompts[default])) |
|
36
|
|
|
return strtobool(answer) |
|
37
|
|
|
except ValueError: |
|
38
|
|
|
if answer == '' and default is not None: |
|
39
|
|
|
return strtobool(default) |
|
40
|
|
|
|
|
41
|
|
|
|
|
42
|
|
|
def chunks(sequence, n): |
|
43
|
|
|
""" Yield successive n-sized chunks from sequence. """ |
|
44
|
|
|
for i in range(0, len(sequence), n): |
|
45
|
|
|
yield sequence[i:i + n] |
|
46
|
|
|
|
|
47
|
|
|
|
|
48
|
|
|
def generate_uid_from_string(value): |
|
49
|
|
|
""" Create unique identifier from a string. """ |
|
50
|
|
|
return hashlib.sha256(value.encode()).hexdigest() |
|
51
|
|
|
|
|
52
|
|
|
|
|
53
|
|
|
def slugify(value): |
|
54
|
|
|
""" |
|
55
|
|
|
Converts to lowercase, removes non-word characters (alphanumerics and |
|
56
|
|
|
underscores) and converts spaces to underscores. Also strips leading and |
|
57
|
|
|
trailing whitespace. |
|
58
|
|
|
|
|
59
|
|
|
Reference |
|
60
|
|
|
--------- |
|
61
|
|
|
https://github.com/django/django/blob/1.7c3/django/utils/text.py#L436 |
|
62
|
|
|
""" |
|
63
|
|
|
try: |
|
64
|
|
|
value = unicode(value, "UTF-8") |
|
65
|
|
|
except NameError: |
|
66
|
|
|
pass # In Python 3 all strings are already unicode. |
|
67
|
|
|
|
|
68
|
|
|
value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') |
|
69
|
|
|
value = re.sub('[^\w\s-]', '', value).strip().lower() |
|
70
|
|
|
return str(re.sub('[-\s]+', '_', value)) |
|
71
|
|
|
|
|
72
|
|
|
|
|
73
|
|
|
def encode_escaped_characters(text, escaping_character="\\"): |
|
74
|
|
|
""" Escape the escaped character using its hex representation """ |
|
75
|
|
|
def hexify(match): |
|
76
|
|
|
# Reference: http://stackoverflow.com/questions/18298251/python-hex-values-to-convert-to-a-string-integer |
|
77
|
|
|
return "\\x" + binascii.hexlify(match.group()[-1].encode()).decode() |
|
78
|
|
|
|
|
79
|
|
|
return re.sub(r"\\.", hexify, text) |
|
80
|
|
|
|
|
81
|
|
|
|
|
82
|
|
|
def decode_escaped_characters(text): |
|
83
|
|
|
""" Convert hex representation to the character it represents """ |
|
84
|
|
|
if len(text) == 0: |
|
85
|
|
|
return '' |
|
86
|
|
|
|
|
87
|
|
|
def unhexify(match): |
|
88
|
|
|
return binascii.unhexlify(match.group()[2:]).decode() |
|
89
|
|
|
|
|
90
|
|
|
return re.sub(r"\\x..", unhexify, text) |
|
91
|
|
|
|
|
92
|
|
|
|
|
93
|
|
|
@contextmanager |
|
94
|
|
|
def open_with_lock(*args, **kwargs): |
|
95
|
|
|
""" Context manager for opening file with an exclusive lock. """ |
|
96
|
|
|
f = open(*args, **kwargs) |
|
97
|
|
|
try: |
|
98
|
|
|
fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) |
|
99
|
|
|
except IOError: |
|
100
|
|
|
logging.info("Can't immediately write-lock the file ({0}), blocking ...".format(f.name)) |
|
101
|
|
|
fcntl.lockf(f, fcntl.LOCK_EX) |
|
102
|
|
|
yield f |
|
103
|
|
|
fcntl.lockf(f, fcntl.LOCK_UN) |
|
104
|
|
|
f.close() |
|
105
|
|
|
|
|
106
|
|
|
|
|
107
|
|
|
def save_dict_to_json_file(path, dictionary): |
|
108
|
|
|
with open(path, "w") as json_file: |
|
109
|
|
|
json_file.write(json.dumps(dictionary, indent=4, separators=(',', ': '))) |
|
110
|
|
|
|
|
111
|
|
|
|
|
112
|
|
|
def load_dict_from_json_file(path): |
|
113
|
|
|
with open(path, "r") as json_file: |
|
114
|
|
|
return json.loads(json_file.read()) |
|
115
|
|
|
|
|
116
|
|
|
|
|
117
|
|
|
def detect_cluster(): |
|
118
|
|
|
# Get server status |
|
119
|
|
|
try: |
|
120
|
|
|
output = Popen(["qstat", "-B"], stdout=PIPE).communicate()[0] |
|
121
|
|
|
if isinstance(output, bytes): |
|
122
|
|
|
output = output.decode("utf-8") |
|
123
|
|
|
except OSError: |
|
124
|
|
|
# If qstat is not available we assume that the cluster is unknown. |
|
125
|
|
|
return None |
|
126
|
|
|
# Get server name from status |
|
127
|
|
|
server_name = output.split('\n')[2].split(' ')[0] |
|
128
|
|
|
# Cleanup the name and return it |
|
129
|
|
|
cluster_name = None |
|
130
|
|
|
if server_name.split('.')[-1] == 'm': |
|
131
|
|
|
cluster_name = "mammouth" |
|
132
|
|
|
elif server_name.split('.')[-1] == 'guil': |
|
133
|
|
|
cluster_name = "guillimin" |
|
134
|
|
|
elif server_name.split('.')[-1] == 'helios': |
|
135
|
|
|
cluster_name = "helios" |
|
136
|
|
|
elif server_name.split('.')[-1] == 'hades': |
|
137
|
|
|
cluster_name = "hades" |
|
138
|
|
|
return cluster_name |
|
139
|
|
|
|
|
140
|
|
|
|
|
141
|
|
|
def get_launcher(cluster_name): |
|
142
|
|
|
if cluster_name == "helios": |
|
143
|
|
|
return "msub" |
|
144
|
|
|
else: |
|
145
|
|
|
return "qsub" |
|
146
|
|
|
|
SMART-Lab /
smartdispatch
