|
1
|
|
|
|
|
2
|
|
|
import argparse |
|
3
|
|
|
import itertools |
|
4
|
|
|
import os |
|
5
|
|
|
import re |
|
6
|
|
|
import tempfile |
|
7
|
|
|
|
|
8
|
|
|
import numpy as np |
|
9
|
|
|
import pandas as pd |
|
10
|
|
|
|
|
11
|
|
|
|
|
12
|
|
|
def reset_first_entry(memory_data): |
|
13
|
|
|
""" |
|
14
|
|
|
Resets the first memory entry for each unique component in the series |
|
15
|
|
|
""" |
|
16
|
|
|
|
|
17
|
|
|
def _reset(s): |
|
18
|
|
|
try: |
|
19
|
|
|
for i in range(64): |
|
20
|
|
|
memory_data.Memory[memory_data.Key.str.contains(s.format(i))].iloc[0][1] = '0' |
|
21
|
|
|
except IndexError: |
|
22
|
|
|
pass |
|
23
|
|
|
|
|
24
|
|
|
_reset("CPU{}") |
|
25
|
|
|
_reset("GPU{}") |
|
26
|
|
|
|
|
27
|
|
|
return memory_data |
|
28
|
|
|
|
|
29
|
|
|
|
|
30
|
|
|
def get_memory_data(frame): |
|
31
|
|
|
memory_data = frame[frame.Message.str.contains("memory usage")] |
|
32
|
|
|
number_finder = re.compile('[0-9]+') |
|
33
|
|
|
memory_data.insert(4, "Memory", [list(itertools.chain(msg.split()[0:1], number_finder.findall(msg))) for msg in |
|
34
|
|
|
memory_data.Message]) |
|
35
|
|
|
|
|
36
|
|
|
memory_data = pd.concat([memory_data.Key, memory_data.Memory], axis=1, join="outer") |
|
37
|
|
|
return reset_first_entry(memory_data) |
|
38
|
|
|
|
|
39
|
|
|
|
|
40
|
|
|
def convert(log_file_list, path, log_level): |
|
41
|
|
|
for log_file in log_file_list: |
|
42
|
|
|
the_key = "" |
|
43
|
|
|
the_interval = 0 # millisecs |
|
44
|
|
|
frame = get_frame(log_file, the_key, 'DEBUG') |
|
45
|
|
|
machine_names = get_machine_names(frame) |
|
46
|
|
|
memory_data = get_memory_data(frame) |
|
47
|
|
|
if log_level != 'DEBUG': |
|
48
|
|
|
frame = get_frame(log_file, the_key, log_level) |
|
49
|
|
|
html_filename = \ |
|
50
|
|
|
set_file_name('/'.join([path, os.path.basename(log_file)])) |
|
51
|
|
|
render_template(frame, machine_names, memory_data, the_interval, html_filename) |
|
52
|
|
|
print("html file created:", html_filename) |
|
53
|
|
|
print("Open the html file in your browser to view the profile.") |
|
54
|
|
|
|
|
55
|
|
|
return frame |
|
|
|
|
|
|
56
|
|
|
|
|
57
|
|
|
|
|
58
|
|
|
def get_frame(log_file, the_key, log_level): |
|
59
|
|
|
import itertools |
|
60
|
|
|
|
|
61
|
|
|
names = ['L', 'Time', 'Machine', 'CPU', 'Type', 'Message'] |
|
62
|
|
|
data = pd.io.parsers.read_fwf(log_file, widths=[2, 13, 6, 6, 7, 1000], |
|
63
|
|
|
names=names) |
|
64
|
|
|
|
|
65
|
|
|
data['Key'] = data['Machine'] + data['CPU'] |
|
66
|
|
|
frame = ((data[data.Type == log_level])[data.columns[[6, 5, 1]]]) |
|
67
|
|
|
frame.insert(0, 'Index', list(range(len(frame)))) |
|
68
|
|
|
frame = frame.sort_values(by=['Key', 'Index']) |
|
69
|
|
|
del frame['Index'] |
|
70
|
|
|
|
|
71
|
|
|
frame.Time = frame.Time.astype(np.int32) |
|
72
|
|
|
startTime = (frame.groupby('Key').first()).Time |
|
73
|
|
|
nElems = frame.groupby('Key').size() |
|
74
|
|
|
|
|
75
|
|
|
shift = [] |
|
76
|
|
|
for i in range(len(nElems)): |
|
77
|
|
|
shift.append([startTime[i]] * nElems[i]) |
|
78
|
|
|
shift = list(itertools.chain(*shift)) |
|
79
|
|
|
|
|
80
|
|
|
frame.Time = frame.Time - shift |
|
81
|
|
|
frame = frame[frame.Key.str.contains(the_key)] |
|
82
|
|
|
frame.insert(3, "Time_end", frame.Time.shift(-1)) |
|
83
|
|
|
frame.Message = frame.Message.str.strip('\n') |
|
84
|
|
|
frame.Message = frame.Message.str.replace("'", "") |
|
85
|
|
|
|
|
86
|
|
|
return frame |
|
87
|
|
|
|
|
88
|
|
|
|
|
89
|
|
|
def get_machine_names(frame): |
|
90
|
|
|
machine_names = frame.copy(deep=True) |
|
91
|
|
|
machine_names = machine_names[frame.Message.str.contains('Rank').replace(np.nan, False)] |
|
92
|
|
|
machine_names.Message = [m.split(':')[-1].strip() for m in frame.Message if isinstance(m, str) and 'Rank' in m] |
|
93
|
|
|
machine_names = machine_names.drop(['Time', 'Time_end'], axis=1) |
|
94
|
|
|
machine_names.Key = [k.split('CPU')[0] for k in machine_names.Key] |
|
95
|
|
|
machine_names.Key = [k.split('GPU')[0] for k in machine_names.Key] |
|
96
|
|
|
machine_names = machine_names.groupby('Key').first() |
|
97
|
|
|
machine_names['Machine'] = machine_names.index.values |
|
98
|
|
|
|
|
99
|
|
|
return machine_names |
|
100
|
|
|
|
|
101
|
|
|
|
|
102
|
|
|
def render_template(frame, machine_names, memory_data, the_interval, outfilename): |
|
103
|
|
|
from jinja2 import Template |
|
104
|
|
|
|
|
105
|
|
|
frame = frame[(frame.Time_end - frame.Time) > the_interval].values |
|
106
|
|
|
|
|
107
|
|
|
f_out = open(outfilename, 'w') |
|
108
|
|
|
dirname = os.path.dirname(__file__) |
|
109
|
|
|
style = os.path.join(dirname, 'style_sheet.css') |
|
110
|
|
|
with open(os.path.join(dirname, 'string_single.html'), 'r') as template_file: |
|
111
|
|
|
template = Template(template_file.read()) |
|
112
|
|
|
|
|
113
|
|
|
f_out.write(template.render(chart_width=1300, position=[16, 9], |
|
114
|
|
|
vals=map(list, frame[:, 0:4]), |
|
115
|
|
|
machines=map(list, machine_names.values), |
|
116
|
|
|
memory=map(list, memory_data.values), |
|
117
|
|
|
style_sheet=style)) |
|
118
|
|
|
f_out.close() |
|
119
|
|
|
|
|
120
|
|
|
return frame |
|
121
|
|
|
|
|
122
|
|
|
|
|
123
|
|
|
def set_file_name(filename): |
|
124
|
|
|
dir_path = os.path.dirname(filename) |
|
125
|
|
|
temp = os.path.basename(filename).split('.') |
|
126
|
|
|
filename = temp[0] + '_' + temp[1] + '.html' |
|
127
|
|
|
outfilename = dir_path + '/' + filename |
|
128
|
|
|
|
|
129
|
|
|
return outfilename |
|
130
|
|
|
|
|
131
|
|
|
|
|
132
|
|
|
def __option_parser(doc=True): |
|
133
|
|
|
""" Option parser for command line arguments. |
|
134
|
|
|
""" |
|
135
|
|
|
parser = argparse.ArgumentParser(prog='savu_profile') |
|
136
|
|
|
|
|
137
|
|
|
parser.add_argument('file', help='Savu output log file') |
|
138
|
|
|
parser.add_argument('-l', '--loglevel', default='INFO', |
|
139
|
|
|
help='Set the log level.') |
|
140
|
|
|
parser.add_argument('-f', '--find', nargs='*', default=[], |
|
141
|
|
|
help='Find lines containing these entries') |
|
142
|
|
|
parser.add_argument('-i', '--ignore', nargs='*', default=[], |
|
143
|
|
|
help='Ignore lines containing these entries') |
|
144
|
|
|
return parser if doc == True else parser.parse_args() |
|
145
|
|
|
|
|
146
|
|
|
|
|
147
|
|
|
def main(): |
|
148
|
|
|
args = __option_parser(doc=False) |
|
149
|
|
|
|
|
150
|
|
|
filename = os.path.abspath(args.file) |
|
151
|
|
|
|
|
152
|
|
|
# create the log file for profiling |
|
153
|
|
|
name, ext = os.path.splitext(os.path.basename(filename)) |
|
154
|
|
|
log_filename = os.path.join(tempfile.mkdtemp(), '{}_{}.log'.format(name, ext)) |
|
155
|
|
|
|
|
156
|
|
|
lfilter = ['L '] + args.find |
|
157
|
|
|
with open(filename, 'r') as finput: |
|
158
|
|
|
with open(log_filename, 'w') as foutput: |
|
159
|
|
|
for line in finput: |
|
160
|
|
|
filter_line = [True if t in line else False for t in lfilter] |
|
161
|
|
|
keep_line = [False if t in line else True for t in args.ignore] |
|
162
|
|
|
line = False if False in filter_line + keep_line else line |
|
163
|
|
|
if line: |
|
164
|
|
|
foutput.write(line) |
|
165
|
|
|
|
|
166
|
|
|
convert([log_filename], os.path.dirname(filename), args.loglevel) |
|
167
|
|
|
|
|
168
|
|
|
|
|
169
|
|
|
if __name__ == "__main__": |
|
170
|
|
|
main() |
|
171
|
|
|
|
DiamondLightSource /
Savu
Loading history...