|
1
|
|
|
import copy |
|
2
|
|
|
from difflib import SequenceMatcher |
|
3
|
|
|
|
|
4
|
|
|
from coalib.results.Diff import ConflictError, Diff |
|
5
|
|
|
from coalib.results.SourceRange import SourceRange |
|
6
|
|
|
|
|
7
|
|
|
|
|
8
|
|
|
def filter_results(original_file_dict, |
|
9
|
|
|
modified_file_dict, |
|
10
|
|
|
original_results, |
|
11
|
|
|
modified_results): |
|
12
|
|
|
""" |
|
13
|
|
|
Filters results for such ones that are unique across file changes |
|
14
|
|
|
|
|
15
|
|
|
:param original_file_dict: Dict of lists of file contents before changes |
|
16
|
|
|
:param modified_file_dict: Dict of lists of file contents after changes |
|
17
|
|
|
:param original_results: List of results of the old files |
|
18
|
|
|
:param modified_results: List of results of the new files |
|
19
|
|
|
:return: List of results from new files that are unique |
|
20
|
|
|
from all those that existed in the old changes |
|
21
|
|
|
""" |
|
22
|
|
|
|
|
23
|
|
|
renamed_files = ensure_files_present(original_file_dict, |
|
24
|
|
|
modified_file_dict) |
|
25
|
|
|
# diffs_dict[file] is a diff between the original and modified file |
|
26
|
|
|
diffs_dict = {} |
|
27
|
|
|
for file in original_file_dict: |
|
28
|
|
|
if file in renamed_files: |
|
29
|
|
|
diffs_dict[file] = Diff.from_string_arrays( |
|
30
|
|
|
original_file_dict[file], |
|
31
|
|
|
modified_file_dict[renamed_files[file]]) |
|
32
|
|
|
else: |
|
33
|
|
|
diffs_dict[file] = Diff.from_string_arrays( |
|
34
|
|
|
original_file_dict[file], |
|
35
|
|
|
modified_file_dict[file]) |
|
36
|
|
|
|
|
37
|
|
|
orig_result_diff_dict_dict = remove_result_ranges_diffs(original_results, |
|
38
|
|
|
original_file_dict) |
|
39
|
|
|
|
|
40
|
|
|
mod_result_diff_dict_dict = remove_result_ranges_diffs(modified_results, |
|
41
|
|
|
modified_file_dict) |
|
42
|
|
|
|
|
43
|
|
|
unique_results = [] |
|
44
|
|
|
|
|
45
|
|
|
for m_r in reversed(modified_results): |
|
46
|
|
|
unique = True |
|
47
|
|
|
|
|
48
|
|
|
for o_r in original_results: |
|
49
|
|
|
|
|
50
|
|
|
if basics_match(o_r, m_r): |
|
51
|
|
|
if source_ranges_match(original_file_dict, |
|
52
|
|
|
diffs_dict, |
|
53
|
|
|
orig_result_diff_dict_dict[o_r], |
|
54
|
|
|
mod_result_diff_dict_dict[m_r], |
|
55
|
|
|
renamed_files): |
|
56
|
|
|
|
|
57
|
|
|
# at least one original result matches completely |
|
58
|
|
|
unique = False |
|
59
|
|
|
break |
|
60
|
|
|
if unique: |
|
61
|
|
|
unique_results.append(m_r) |
|
62
|
|
|
|
|
63
|
|
|
return unique_results |
|
64
|
|
|
|
|
65
|
|
|
|
|
66
|
|
|
def basics_match(original_result, |
|
67
|
|
|
modified_result): |
|
68
|
|
|
""" |
|
69
|
|
|
Checks whether the following properties of two results match: |
|
70
|
|
|
* origin |
|
71
|
|
|
* message |
|
72
|
|
|
* severity |
|
73
|
|
|
* debug_msg |
|
74
|
|
|
|
|
75
|
|
|
:param original_result: A result of the old files |
|
76
|
|
|
:param modified_result: A result of the new files |
|
77
|
|
|
:return: Boolean value whether or not the properties match |
|
78
|
|
|
""" |
|
79
|
|
|
|
|
80
|
|
|
return all(getattr(original_result, member) == |
|
81
|
|
|
getattr(modified_result, member) |
|
82
|
|
|
for member in ['origin', 'message', 'severity', 'debug_msg']) |
|
83
|
|
|
|
|
84
|
|
|
|
|
85
|
|
|
def source_ranges_match(original_file_dict, |
|
86
|
|
|
diff_dict, |
|
87
|
|
|
original_result_diff_dict, |
|
88
|
|
|
modified_result_diff_dict, |
|
89
|
|
|
renamed_files): |
|
90
|
|
|
""" |
|
91
|
|
|
Checks whether the SourceRanges of two results match |
|
92
|
|
|
|
|
93
|
|
|
:param original_file_dict: Dict of lists of file contents before changes |
|
94
|
|
|
:param diff_dict: Dict of diffs describing the changes per file |
|
95
|
|
|
:param original_result_diff_dict: diff for each file for this result |
|
96
|
|
|
:param modified_result_diff_dict: guess |
|
97
|
|
|
:param renamed_files: A dictionary containing file renamings across runs |
|
98
|
|
|
:return: Boolean value whether the SourceRanges match |
|
99
|
|
|
""" |
|
100
|
|
|
for file_name in original_file_dict: |
|
101
|
|
|
mod_file_name = file_name |
|
102
|
|
|
if file_name in renamed_files: |
|
103
|
|
|
mod_file_name = renamed_files[file_name] |
|
104
|
|
|
|
|
105
|
|
|
try: # fails if the affected range of the result get's modified |
|
106
|
|
|
original_total_diff = (diff_dict[file_name] + |
|
107
|
|
|
original_result_diff_dict[file_name]) |
|
108
|
|
|
except ConflictError: |
|
109
|
|
|
return False |
|
110
|
|
|
|
|
111
|
|
|
# original file with file_diff and original_diff applied |
|
112
|
|
|
original_total_file = original_total_diff.modified |
|
113
|
|
|
# modified file with modified_diff applied |
|
114
|
|
|
modified_total_file = modified_result_diff_dict[mod_file_name].modified |
|
115
|
|
|
if original_total_file != modified_total_file: |
|
116
|
|
|
return False |
|
117
|
|
|
return True |
|
118
|
|
|
|
|
119
|
|
|
|
|
120
|
|
|
def remove_range(file_contents, source_range): |
|
121
|
|
|
""" |
|
122
|
|
|
removes the chars covered by the sourceRange from the file |
|
123
|
|
|
|
|
124
|
|
|
:param file_contents: list of lines in the file |
|
125
|
|
|
:param source_range: Source Range |
|
126
|
|
|
:return: list of file contents without specified chars removed |
|
127
|
|
|
""" |
|
128
|
|
|
if not file_contents: |
|
129
|
|
|
return [] |
|
130
|
|
|
|
|
131
|
|
|
newfile = list(file_contents) |
|
132
|
|
|
# attention: line numbers in the SourceRange are human-readable, |
|
133
|
|
|
# list indices start with 0 |
|
134
|
|
|
|
|
135
|
|
|
source_range = source_range.expand(file_contents) |
|
136
|
|
|
|
|
137
|
|
|
if source_range.start.line == source_range.end.line: |
|
138
|
|
|
# if it's all in one line, replace the line by it's beginning and end |
|
139
|
|
|
newfile[source_range.start.line - 1] = ( |
|
140
|
|
|
newfile[source_range.start.line - 1][:source_range.start.column-1] |
|
141
|
|
|
+ newfile[source_range.start.line - 1][source_range.end.column:]) |
|
142
|
|
|
if newfile[source_range.start.line - 1] == "": |
|
143
|
|
|
del newfile[source_range.start.line - 1] |
|
144
|
|
|
else: |
|
145
|
|
|
# cut away after start |
|
146
|
|
|
newfile[source_range.start.line - 1] = ( |
|
147
|
|
|
newfile[source_range.start.line - 1][:source_range.start.column-1]) |
|
148
|
|
|
|
|
149
|
|
|
# cut away before end |
|
150
|
|
|
newfile[source_range.end.line - 1] = ( |
|
151
|
|
|
newfile[source_range.end.line - 1][source_range.end.column:]) |
|
152
|
|
|
|
|
153
|
|
|
# start: index = first line number ==> line after first line |
|
154
|
|
|
# end: index = last line -2 ==> line before last line |
|
155
|
|
|
|
|
156
|
|
|
for i in reversed(range( |
|
157
|
|
|
source_range.start.line, source_range.end.line - 1)): |
|
158
|
|
|
del newfile[i] |
|
159
|
|
|
|
|
160
|
|
|
# remove leftover empty lines |
|
161
|
|
|
# the first line here is actually the former `source_range.end.line -1` |
|
162
|
|
|
if newfile[source_range.start.line] == "": |
|
163
|
|
|
del newfile[source_range.start.line] |
|
164
|
|
|
if newfile[source_range.start.line - 1] == "": |
|
165
|
|
|
del newfile[source_range.start.line - 1] |
|
166
|
|
|
|
|
167
|
|
|
return newfile |
|
168
|
|
|
|
|
169
|
|
|
|
|
170
|
|
|
def remove_result_ranges_diffs(result_list, file_dict): |
|
171
|
|
|
""" |
|
172
|
|
|
Calculates the diffs to all files in file_dict that describe the removal of |
|
173
|
|
|
each respective result's affected code. |
|
174
|
|
|
|
|
175
|
|
|
:param result_list: list of results |
|
176
|
|
|
:param file_dict: dict of file contents |
|
177
|
|
|
:return: returnvalue[result][file] is a diff of the changes the |
|
178
|
|
|
removal of this result's affected code would cause for |
|
179
|
|
|
the file. |
|
180
|
|
|
""" |
|
181
|
|
|
result_diff_dict_dict = {} |
|
182
|
|
|
for original_result in result_list: |
|
183
|
|
|
mod_file_dict = copy.deepcopy(file_dict) |
|
184
|
|
|
|
|
185
|
|
|
# gather all source ranges from this result |
|
186
|
|
|
source_ranges = [] |
|
187
|
|
|
|
|
188
|
|
|
# SourceRanges must be sorted backwards and overlaps must be eliminated |
|
189
|
|
|
# this way, the deletion based on sourceRanges is not offset by |
|
190
|
|
|
# previous deletions in the same line that invalidate the indices. |
|
191
|
|
|
previous = None |
|
192
|
|
|
|
|
193
|
|
|
for source_range in sorted(original_result.affected_code, reverse=True): |
|
194
|
|
|
# previous exists and overlaps |
|
195
|
|
|
if previous is not None and source_range.overlaps(previous): |
|
196
|
|
|
combined_sr = SourceRange.join(previous, source_range) |
|
197
|
|
|
previous = combined_sr |
|
198
|
|
|
elif previous is None: |
|
199
|
|
|
previous = source_range |
|
200
|
|
|
# previous exists but it doesn't overlap |
|
201
|
|
|
else: |
|
202
|
|
|
source_ranges.append(previous) |
|
203
|
|
|
previous = source_range |
|
204
|
|
|
# don't forget last entry if there were any: |
|
205
|
|
|
if previous: |
|
206
|
|
|
source_ranges.append(previous) |
|
207
|
|
|
|
|
208
|
|
|
for source_range in source_ranges: |
|
209
|
|
|
file_name = source_range.file |
|
210
|
|
|
new_file = remove_range(mod_file_dict[file_name], |
|
211
|
|
|
source_range) |
|
212
|
|
|
mod_file_dict[file_name] = new_file |
|
213
|
|
|
|
|
214
|
|
|
diff_dict = {} |
|
215
|
|
|
for file_name in file_dict: |
|
216
|
|
|
diff_dict[file_name] = Diff.from_string_arrays( |
|
217
|
|
|
file_dict[file_name], |
|
218
|
|
|
mod_file_dict[file_name]) |
|
219
|
|
|
|
|
220
|
|
|
result_diff_dict_dict[original_result] = diff_dict |
|
221
|
|
|
|
|
222
|
|
|
return result_diff_dict_dict |
|
223
|
|
|
|
|
224
|
|
|
|
|
225
|
|
|
def ensure_files_present(original_file_dict, modified_file_dict): |
|
226
|
|
|
""" |
|
227
|
|
|
Ensures that all files are available as keys in both dicts. Return a |
|
228
|
|
|
dictionary of renamed files. |
|
229
|
|
|
|
|
230
|
|
|
:param original_file_dict: Dict of lists of file contents before changes |
|
231
|
|
|
:param modified_file_dict: Dict of lists of file contents after changes |
|
232
|
|
|
""" |
|
233
|
|
|
affected_files = set(original_file_dict.keys()).union( |
|
234
|
|
|
set(modified_file_dict.keys())) |
|
235
|
|
|
original_unique_files = affected_files - set(modified_file_dict.keys()) |
|
236
|
|
|
renamed_files_dict = {} |
|
237
|
|
|
for file in affected_files: |
|
238
|
|
|
if file not in original_file_dict: |
|
239
|
|
|
renamed = 0 |
|
240
|
|
|
for comparable_file in original_unique_files: |
|
241
|
|
|
s = SequenceMatcher( |
|
242
|
|
|
None, |
|
243
|
|
|
''.join(modified_file_dict[file]), |
|
244
|
|
|
''.join(original_file_dict[comparable_file])) |
|
245
|
|
|
if s.real_quick_ratio() >= 0.5: |
|
246
|
|
|
if s.ratio() > 0.5: |
|
247
|
|
|
renamed_files_dict[comparable_file] = file |
|
248
|
|
|
renamed = 1 |
|
249
|
|
|
break |
|
250
|
|
|
if renamed == 0: |
|
251
|
|
|
original_file_dict[file] = [] |
|
252
|
|
|
if file not in modified_file_dict: |
|
253
|
|
|
modified_file_dict[file] = [] |
|
254
|
|
|
return renamed_files_dict |
|
255
|
|
|
|
coala-analyzer /
coala
