1 | #!/usr/bin/env python2.7 |
||
2 | # Licensed to the StackStorm, Inc ('StackStorm') under one or more |
||
3 | # contributor license agreements. See the NOTICE file distributed with |
||
4 | # this work for additional information regarding copyright ownership. |
||
5 | # The ASF licenses this file to You under the Apache License, Version 2.0 |
||
6 | # (the "License"); you may not use this file except in compliance with |
||
7 | # the License. You may obtain a copy of the License at |
||
8 | # |
||
9 | # http://www.apache.org/licenses/LICENSE-2.0 |
||
10 | # |
||
11 | # Unless required by applicable law or agreed to in writing, software |
||
12 | # distributed under the License is distributed on an "AS IS" BASIS, |
||
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||
14 | # See the License for the specific language governing permissions and |
||
15 | # limitations under the License. |
||
16 | |||
17 | from __future__ import print_function |
||
18 | from __future__ import absolute_import |
||
19 | import collections |
||
20 | import fnmatch |
||
21 | import os |
||
22 | import re |
||
23 | import sys |
||
24 | |||
25 | from tabulate import tabulate |
||
26 | import six |
||
27 | |||
28 | LOG_ALERT_PERCENT = 5 # default. |
||
29 | |||
30 | EVILS = [ |
||
31 | 'info', |
||
32 | 'debug', |
||
33 | 'warning', |
||
34 | 'exception', |
||
35 | 'error', |
||
36 | 'audit' |
||
37 | ] |
||
38 | |||
39 | LOG_VARS = [ |
||
40 | 'LOG', |
||
41 | 'Log', |
||
42 | 'log', |
||
43 | 'LOGGER', |
||
44 | 'Logger', |
||
45 | 'logger', |
||
46 | 'logging', |
||
47 | 'LOGGING' |
||
48 | ] |
||
49 | |||
50 | FILE_LOG_COUNT = collections.defaultdict() |
||
51 | FILE_LINE_COUNT = collections.defaultdict() |
||
52 | |||
53 | |||
54 | def _parse_args(args): |
||
55 | global LOG_ALERT_PERCENT |
||
56 | params = {} |
||
57 | if len(args) > 1: |
||
58 | params['alert_percent'] = args[1] |
||
59 | LOG_ALERT_PERCENT = int(args[1]) |
||
60 | return params |
||
61 | |||
62 | |||
63 | def _skip_file(filename): |
||
64 | if filename.startswith('.') or filename.startswith('_'): |
||
65 | return True |
||
66 | |||
67 | |||
68 | def _get_files(dir_path): |
||
69 | if not os.path.exists(dir_path): |
||
70 | print('Directory %s doesn\'t exist.' % dir_path) |
||
71 | |||
72 | files = [] |
||
73 | exclude = set(['virtualenv', 'build', '.tox']) |
||
74 | for root, dirnames, filenames in os.walk(dir_path): |
||
75 | dirnames[:] = [d for d in dirnames if d not in exclude] |
||
76 | for filename in fnmatch.filter(filenames, '*.py'): |
||
77 | if not _skip_file(filename): |
||
78 | files.append(os.path.join(root, filename)) |
||
79 | return files |
||
80 | |||
81 | |||
82 | # TODO: Regex compiling will be faster but I cannot get it to work :( |
||
83 | def _build_regex(): |
||
84 | regex_strings = {} |
||
85 | regexes = {} |
||
86 | for level in EVILS: |
||
87 | regex_string = '|'.join(['\.'.join([log, level]) for log in LOG_VARS]) |
||
0 ignored issues
–
show
|
|||
88 | regex_strings[level] = regex_string |
||
89 | # print('Level: %s, regex_string: %s' % (level, regex_strings[level])) |
||
90 | regexes[level] = re.compile(regex_strings[level]) |
||
91 | return regexes |
||
92 | |||
93 | |||
94 | def _regex_match(line, regexes): |
||
95 | pass |
||
96 | |||
97 | |||
98 | def _build_str_matchers(): |
||
99 | match_strings = {} |
||
100 | for level in EVILS: |
||
101 | match_strings[level] = ['.'.join([log, level]) for log in LOG_VARS] |
||
102 | return match_strings |
||
103 | |||
104 | |||
105 | def _get_log_count_dict(): |
||
106 | return [(level, 0) for level in EVILS] |
||
107 | |||
108 | |||
109 | def _alert(fil, lines, logs, logs_level): |
||
110 | print('WARNING: Too many logs!!!: File: %s, total lines: %d, log lines: %d, percent: %f, ' |
||
111 | 'logs: %s' % (fil, lines, logs, float(logs) / lines * 100, logs_level)) |
||
112 | |||
113 | |||
114 | def _match(line, match_strings): |
||
115 | for level, match_strings in six.iteritems(match_strings): |
||
116 | for match_string in match_strings: |
||
117 | if line.startswith(match_string): |
||
118 | # print('Line: %s, match: %s' % (line, match_string)) |
||
119 | return True, level, line |
||
120 | return False, 'UNKNOWN', line |
||
121 | |||
122 | |||
123 | def _detect_log_lines(fil, matchers): |
||
124 | global FILE_LOG_COUNT |
||
125 | FILE_LOG_COUNT[fil] = dict(_get_log_count_dict()) |
||
126 | # print('Working on file: %s' % fil) |
||
127 | with open(fil) as f: |
||
128 | lines = f.readlines() |
||
129 | FILE_LINE_COUNT[fil] = len(lines) |
||
130 | |||
131 | ln = 0 |
||
132 | for line in lines: |
||
133 | line = line.strip() |
||
134 | ln += 1 |
||
135 | matched, level, line = _match(line, matchers) |
||
136 | if matched: |
||
137 | # print('File: %s, Level: %s, Line: %d:%s' % (fil, level, ln, line.strip())) |
||
138 | FILE_LOG_COUNT[fil][level] += 1 |
||
139 | |||
140 | |||
141 | def _post_process(file_dir): |
||
142 | alerts = [] |
||
143 | for fil, lines in six.iteritems(FILE_LINE_COUNT): |
||
144 | log_lines_count_level = FILE_LOG_COUNT[fil] |
||
145 | total_log_count = 0 |
||
146 | for level, count in six.iteritems(log_lines_count_level): |
||
147 | total_log_count += count |
||
148 | if total_log_count > 0: |
||
149 | if float(total_log_count) / lines * 100 > LOG_ALERT_PERCENT: |
||
150 | if file_dir in fil: |
||
151 | fil = fil[len(file_dir) + 1:] |
||
152 | alerts.append([fil, lines, total_log_count, float(total_log_count) / lines * 100, |
||
153 | log_lines_count_level['audit'], |
||
154 | log_lines_count_level['exception'], |
||
155 | log_lines_count_level['error'], |
||
156 | log_lines_count_level['warning'], |
||
157 | log_lines_count_level['info'], |
||
158 | log_lines_count_level['debug']]) |
||
159 | # sort by percent |
||
160 | alerts.sort(key=lambda alert: alert[3], reverse=True) |
||
161 | print(tabulate(alerts, headers=['File', 'Lines', 'Logs', 'Percent', 'adt', 'exc', 'err', 'wrn', |
||
162 | 'inf', 'dbg'])) |
||
163 | |||
164 | |||
165 | def main(args): |
||
166 | params = _parse_args(args) |
||
167 | file_dir = params.get('dir', os.getcwd()) |
||
168 | files = _get_files(file_dir) |
||
169 | matchers = _build_str_matchers() |
||
170 | for f in files: |
||
171 | _detect_log_lines(f, matchers) |
||
172 | _post_process(file_dir) |
||
173 | |||
174 | |||
175 | if __name__ == '__main__': |
||
176 | main(sys.argv) |
||
177 |
Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with
r
orR
are they interpreted as regular expressions.The escape sequence that was used indicates that you might have intended to write a regular expression.
Learn more about the available escape sequences. in the Python documentation.