1 | #!/usr/bin/env python2.7 |
||
2 | # Licensed to the StackStorm, Inc ('StackStorm') under one or more |
||
3 | # contributor license agreements. See the NOTICE file distributed with |
||
4 | # this work for additional information regarding copyright ownership. |
||
5 | # The ASF licenses this file to You under the Apache License, Version 2.0 |
||
6 | # (the "License"); you may not use this file except in compliance with |
||
7 | # the License. You may obtain a copy of the License at |
||
8 | # |
||
9 | # http://www.apache.org/licenses/LICENSE-2.0 |
||
10 | # |
||
11 | # Unless required by applicable law or agreed to in writing, software |
||
12 | # distributed under the License is distributed on an "AS IS" BASIS, |
||
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||
14 | # See the License for the specific language governing permissions and |
||
15 | # limitations under the License. |
||
16 | |||
17 | from __future__ import print_function |
||
18 | from __future__ import absolute_import |
||
19 | import collections |
||
20 | import fnmatch |
||
21 | import os |
||
22 | import re |
||
23 | import sys |
||
24 | |||
25 | from tabulate import tabulate |
||
26 | import six |
||
27 | |||
28 | LOG_ALERT_PERCENT = 5 # default. |
||
29 | |||
30 | EVILS = [ |
||
31 | 'info', |
||
32 | 'debug', |
||
33 | 'warning', |
||
34 | 'exception', |
||
35 | 'error', |
||
36 | 'audit' |
||
37 | ] |
||
38 | |||
39 | LOG_VARS = [ |
||
40 | 'LOG', |
||
41 | 'Log', |
||
42 | 'log', |
||
43 | 'LOGGER', |
||
44 | 'Logger', |
||
45 | 'logger', |
||
46 | 'logging', |
||
47 | 'LOGGING' |
||
48 | ] |
||
49 | |||
50 | FILE_LOG_COUNT = collections.defaultdict() |
||
51 | FILE_LINE_COUNT = collections.defaultdict() |
||
52 | |||
53 | |||
54 | def _parse_args(args): |
||
55 | global LOG_ALERT_PERCENT |
||
0 ignored issues
–
show
|
|||
56 | params = {} |
||
57 | if len(args) > 1: |
||
58 | params['alert_percent'] = args[1] |
||
59 | LOG_ALERT_PERCENT = int(args[1]) |
||
60 | return params |
||
61 | |||
62 | |||
63 | def _skip_file(filename): |
||
64 | if filename.startswith('.') or filename.startswith('_'): |
||
65 | return True |
||
66 | |||
67 | |||
68 | def _get_files(dir_path): |
||
69 | if not os.path.exists(dir_path): |
||
70 | print('Directory %s doesn\'t exist.' % dir_path) |
||
71 | |||
72 | files = [] |
||
73 | exclude = set(['virtualenv', 'build', '.tox']) |
||
74 | for root, dirnames, filenames in os.walk(dir_path): |
||
75 | dirnames[:] = [d for d in dirnames if d not in exclude] |
||
76 | for filename in fnmatch.filter(filenames, '*.py'): |
||
77 | if not _skip_file(filename): |
||
78 | files.append(os.path.join(root, filename)) |
||
79 | return files |
||
80 | |||
81 | |||
82 | # TODO: Regex compiling will be faster but I cannot get it to work :( |
||
83 | def _build_regex(): |
||
84 | regex_strings = {} |
||
85 | regexes = {} |
||
86 | for level in EVILS: |
||
87 | regex_string = '|'.join(['\.'.join([log, level]) for log in LOG_VARS]) |
||
0 ignored issues
–
show
A suspicious escape sequence
\. was found. Did you maybe forget to add an r prefix?
Escape sequences in Python are generally interpreted according to rules similar
to standard C. Only if strings are prefixed with The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation.
Loading history...
|
|||
88 | regex_strings[level] = regex_string |
||
89 | # print('Level: %s, regex_string: %s' % (level, regex_strings[level])) |
||
90 | regexes[level] = re.compile(regex_strings[level]) |
||
91 | return regexes |
||
92 | |||
93 | |||
94 | def _regex_match(line, regexes): |
||
95 | pass |
||
96 | |||
97 | |||
98 | def _build_str_matchers(): |
||
99 | match_strings = {} |
||
100 | for level in EVILS: |
||
101 | match_strings[level] = ['.'.join([log, level]) for log in LOG_VARS] |
||
102 | return match_strings |
||
103 | |||
104 | |||
105 | def _get_log_count_dict(): |
||
106 | return [(level, 0) for level in EVILS] |
||
107 | |||
108 | |||
109 | def _alert(fil, lines, logs, logs_level): |
||
110 | print('WARNING: Too many logs!!!: File: %s, total lines: %d, log lines: %d, percent: %f, ' |
||
111 | 'logs: %s' % (fil, lines, logs, float(logs) / lines * 100, logs_level)) |
||
112 | |||
113 | |||
114 | def _match(line, match_strings): |
||
115 | for level, match_strings in six.iteritems(match_strings): |
||
116 | for match_string in match_strings: |
||
117 | if line.startswith(match_string): |
||
118 | # print('Line: %s, match: %s' % (line, match_string)) |
||
119 | return True, level, line |
||
120 | return False, 'UNKNOWN', line |
||
121 | |||
122 | |||
123 | def _detect_log_lines(fil, matchers): |
||
124 | global FILE_LOG_COUNT |
||
0 ignored issues
–
show
|
|||
125 | FILE_LOG_COUNT[fil] = dict(_get_log_count_dict()) |
||
126 | # print('Working on file: %s' % fil) |
||
127 | with open(fil) as f: |
||
128 | lines = f.readlines() |
||
129 | FILE_LINE_COUNT[fil] = len(lines) |
||
130 | |||
131 | ln = 0 |
||
132 | for line in lines: |
||
133 | line = line.strip() |
||
134 | ln += 1 |
||
135 | matched, level, line = _match(line, matchers) |
||
136 | if matched: |
||
137 | # print('File: %s, Level: %s, Line: %d:%s' % (fil, level, ln, line.strip())) |
||
138 | FILE_LOG_COUNT[fil][level] += 1 |
||
139 | |||
140 | |||
141 | def _post_process(file_dir): |
||
142 | alerts = [] |
||
143 | for fil, lines in six.iteritems(FILE_LINE_COUNT): |
||
144 | log_lines_count_level = FILE_LOG_COUNT[fil] |
||
145 | total_log_count = 0 |
||
146 | for level, count in six.iteritems(log_lines_count_level): |
||
147 | total_log_count += count |
||
148 | if total_log_count > 0: |
||
149 | if float(total_log_count) / lines * 100 > LOG_ALERT_PERCENT: |
||
150 | if file_dir in fil: |
||
151 | fil = fil[len(file_dir) + 1:] |
||
152 | alerts.append([fil, lines, total_log_count, float(total_log_count) / lines * 100, |
||
153 | log_lines_count_level['audit'], |
||
154 | log_lines_count_level['exception'], |
||
155 | log_lines_count_level['error'], |
||
156 | log_lines_count_level['warning'], |
||
157 | log_lines_count_level['info'], |
||
158 | log_lines_count_level['debug']]) |
||
159 | # sort by percent |
||
160 | alerts.sort(key=lambda alert: alert[3], reverse=True) |
||
161 | print(tabulate(alerts, headers=['File', 'Lines', 'Logs', 'Percent', 'adt', 'exc', 'err', 'wrn', |
||
162 | 'inf', 'dbg'])) |
||
163 | |||
164 | |||
165 | def main(args): |
||
166 | params = _parse_args(args) |
||
167 | file_dir = params.get('dir', os.getcwd()) |
||
168 | files = _get_files(file_dir) |
||
169 | matchers = _build_str_matchers() |
||
170 | for f in files: |
||
171 | _detect_log_lines(f, matchers) |
||
172 | _post_process(file_dir) |
||
173 | |||
174 | |||
175 | if __name__ == '__main__': |
||
176 | main(sys.argv) |
||
177 |
Usage of
global
can make code hard to read and test, its usage is generally not recommended unless you are dealing with legacy code.