1
|
|
|
#!/usr/bin/python3 |
2
|
|
|
|
3
|
|
|
from __future__ import print_function |
4
|
|
|
|
5
|
|
|
import sys |
6
|
|
|
import os |
7
|
|
|
import jinja2 |
8
|
|
|
import argparse |
9
|
|
|
import json |
10
|
|
|
import re |
11
|
|
|
import random |
12
|
|
|
|
13
|
|
|
from ssg import yaml, cce, products |
14
|
|
|
from ssg.shims import input_func |
15
|
|
|
from ssg.utils import read_file_list |
16
|
|
|
import ssg |
17
|
|
|
import ssg.products |
18
|
|
|
import ssg.rules |
19
|
|
|
import ssg.rule_yaml |
20
|
|
|
|
21
|
|
|
|
22
|
|
|
SSG_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) |
23
|
|
|
TO_SORT = ['identifiers', 'references'] |
24
|
|
|
|
25
|
|
|
|
26
|
|
|
_COMMANDS = dict() |
27
|
|
|
|
28
|
|
|
|
29
|
|
|
def command(name, description): |
30
|
|
|
def wrapper(wrapped): |
31
|
|
|
_COMMANDS[name] = wrapped |
32
|
|
|
wrapped.description = description |
33
|
|
|
return wrapped |
34
|
|
|
return wrapper |
35
|
|
|
|
36
|
|
|
|
37
|
|
|
def has_empty_identifier(rule_path, rule, rule_lines): |
38
|
|
|
if 'identifiers' in rule and rule['identifiers'] is None: |
39
|
|
|
return True |
40
|
|
|
|
41
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
42
|
|
|
for _, value in rule['identifiers'].items(): |
43
|
|
|
if str(value).strip() == "": |
44
|
|
|
return True |
45
|
|
|
return False |
46
|
|
|
|
47
|
|
|
|
48
|
|
|
def has_no_cce(yaml_file, product_yaml=None): |
49
|
|
|
rule = yaml.open_and_macro_expand(yaml_file, product_yaml) |
50
|
|
|
product = product_yaml["product"] |
51
|
|
|
if "prodtype" in rule and product not in rule["prodtype"]: |
52
|
|
|
return False |
53
|
|
|
if 'identifiers' in rule and rule['identifiers'] is None: |
54
|
|
|
return True |
55
|
|
|
|
56
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
57
|
|
|
for ident in rule['identifiers']: |
58
|
|
|
if ident == "cce@" + product: |
59
|
|
|
return False |
60
|
|
|
return True |
61
|
|
|
|
62
|
|
|
|
63
|
|
|
def has_empty_references(rule_path, rule, rule_lines): |
64
|
|
|
if 'references' in rule and rule['references'] is None: |
65
|
|
|
return True |
66
|
|
|
|
67
|
|
|
if 'references' in rule and rule['references'] is not None: |
68
|
|
|
for _, value in rule['references'].items(): |
69
|
|
|
if str(value).strip() == "": |
70
|
|
|
return True |
71
|
|
|
return False |
72
|
|
|
|
73
|
|
|
|
74
|
|
|
def has_prefix_cce(rule_path, rule, rule_lines): |
75
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
76
|
|
|
for i_type, i_value in rule['identifiers'].items(): |
77
|
|
|
if i_type[0:3] == 'cce': |
78
|
|
|
has_prefix = i_value[0:3].upper() == 'CCE' |
79
|
|
|
remainder_valid = cce.is_cce_format_valid("CCE-" + i_value[3:]) |
80
|
|
|
remainder_valid |= cce.is_cce_format_valid("CCE-" + i_value[4:]) |
81
|
|
|
return has_prefix and remainder_valid |
82
|
|
|
return False |
83
|
|
|
|
84
|
|
|
|
85
|
|
|
def has_invalid_cce(rule_path, rule, rule_lines): |
86
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
87
|
|
|
for i_type, i_value in rule['identifiers'].items(): |
88
|
|
|
if i_type[0:3] == 'cce': |
89
|
|
|
if not cce.is_cce_value_valid("CCE-" + str(i_value)): |
90
|
|
|
return True |
91
|
|
|
return False |
92
|
|
|
|
93
|
|
|
|
94
|
|
|
def has_int_identifier(rule_path, rule, rule_lines): |
95
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
96
|
|
|
for _, value in rule['identifiers'].items(): |
97
|
|
|
if type(value) != str: |
98
|
|
|
return True |
99
|
|
|
return False |
100
|
|
|
|
101
|
|
|
|
102
|
|
|
def has_int_reference(rule_path, rule, rule_lines): |
103
|
|
|
if 'references' in rule and rule['references'] is not None: |
104
|
|
|
for _, value in rule['references'].items(): |
105
|
|
|
if type(value) != str: |
106
|
|
|
return True |
107
|
|
|
return False |
108
|
|
|
|
109
|
|
|
|
110
|
|
|
def has_duplicated_subkeys(rule_path, rule, rule_lines): |
111
|
|
|
return ssg.rule_yaml.has_duplicated_subkeys(rule_path, rule_lines, TO_SORT) |
112
|
|
|
|
113
|
|
|
|
114
|
|
|
def has_unordered_sections(rule_path, rule, rule_lines): |
115
|
|
|
if 'references' in rule or 'identifiers' in rule: |
116
|
|
|
new_lines = ssg.rule_yaml.sort_section_keys(rule_path, rule_lines, TO_SORT) |
117
|
|
|
|
118
|
|
|
# Compare string representations to avoid issues with references being |
119
|
|
|
# different. |
120
|
|
|
return "\n".join(rule_lines) != "\n".join(new_lines) |
121
|
|
|
|
122
|
|
|
return False |
123
|
|
|
|
124
|
|
|
|
125
|
|
|
def rule_data_generator(args): |
126
|
|
|
# Iterates over all know rules in the build system (according to |
127
|
|
|
# rule_dir_json.py) and attempts to load the resulting YAML files. |
128
|
|
|
# If they parse correctly, yield them as a result. |
129
|
|
|
# |
130
|
|
|
# Note: this has become a generator rather than returning a list of |
131
|
|
|
# results. |
132
|
|
|
|
133
|
|
|
product_yamls = dict() |
134
|
|
|
|
135
|
|
|
rule_dirs = json.load(open(args.json)) |
136
|
|
|
for rule_id in rule_dirs: |
137
|
|
|
rule_obj = rule_dirs[rule_id] |
138
|
|
|
|
139
|
|
|
if 'products' not in rule_obj or not rule_obj['products']: |
140
|
|
|
print(rule_id, rule_obj) |
141
|
|
|
assert rule_obj['products'] |
142
|
|
|
product = rule_obj['products'][0] |
143
|
|
|
|
144
|
|
|
if product not in product_yamls: |
145
|
|
|
product_path = ssg.products.product_yaml_path(args.root, product) |
146
|
|
|
product_yaml = ssg.products.load_product_yaml(product_path) |
147
|
|
|
properties_directory = os.path.join(args.root, "product_properties") |
148
|
|
|
product_yaml.read_properties_from_directory(properties_directory) |
149
|
|
|
product_yamls[product] = product_yaml |
150
|
|
|
|
151
|
|
|
local_env_yaml = dict(cmake_build_type='Debug') |
152
|
|
|
local_env_yaml.update(product_yamls[product]) |
153
|
|
|
local_env_yaml['rule_id'] = rule_id |
154
|
|
|
|
155
|
|
|
rule_path = ssg.rules.get_rule_dir_yaml(rule_obj['dir']) |
156
|
|
|
try: |
157
|
|
|
rule = yaml.open_and_macro_expand(rule_path, local_env_yaml) |
158
|
|
|
rule_lines = read_file_list(rule_path) |
159
|
|
|
yield rule_path, rule, rule_lines, product_path, local_env_yaml |
|
|
|
|
160
|
|
|
except jinja2.exceptions.UndefinedError as ue: |
161
|
|
|
msg = "Failed to parse file {0} (with product.yml: {1}). Skipping. {2}" |
162
|
|
|
msg = msg.format(rule_path, product_path, ue) |
163
|
|
|
print(msg, file=sys.stderr) |
164
|
|
|
|
165
|
|
|
|
166
|
|
|
def find_rules_generator(args, func): |
167
|
|
|
for item in rule_data_generator(args): |
168
|
|
|
rule_path, rule, rule_lines, product_path, local_env_yaml = item |
169
|
|
|
if func(rule_path, rule, rule_lines): |
170
|
|
|
yield (rule_path, product_path, local_env_yaml) |
171
|
|
|
|
172
|
|
|
|
173
|
|
|
def find_rules(args, func): |
174
|
|
|
# Returns find_rules_generator as a list |
175
|
|
|
return list(find_rules_generator(args, func)) |
176
|
|
|
|
177
|
|
|
|
178
|
|
|
def print_file(file_contents): |
179
|
|
|
for line_num in range(0, len(file_contents)): |
180
|
|
|
print("%d: %s" % (line_num, file_contents[line_num])) |
181
|
|
|
|
182
|
|
|
|
183
|
|
|
def find_section_lines(file_contents, sec): |
184
|
|
|
# Hack to find a global key ("section"/sec) in a YAML-like file. |
185
|
|
|
# All indented lines until the next global key are included in the range. |
186
|
|
|
# For example: |
187
|
|
|
# |
188
|
|
|
# 0: not_it: |
189
|
|
|
# 1: - value |
190
|
|
|
# 2: this_one: |
191
|
|
|
# 3: - 2 |
192
|
|
|
# 4: - 5 |
193
|
|
|
# 5: |
194
|
|
|
# 6: nor_this: |
195
|
|
|
# |
196
|
|
|
# for the section "this_one", the result [(2, 5)] will be returned. |
197
|
|
|
# Note that multiple sections may exist in a file and each will be |
198
|
|
|
# identified and returned. |
199
|
|
|
sec_ranges = [] |
200
|
|
|
|
201
|
|
|
sec_id = sec + ":" |
202
|
|
|
sec_len = len(sec_id) |
203
|
|
|
end_num = len(file_contents) |
204
|
|
|
line_num = 0 |
205
|
|
|
|
206
|
|
View Code Duplication |
while line_num < end_num: |
|
|
|
|
207
|
|
|
if len(file_contents[line_num]) >= sec_len: |
208
|
|
|
if file_contents[line_num][0:sec_len] == sec_id: |
209
|
|
|
begin = line_num |
210
|
|
|
line_num += 1 |
211
|
|
|
while line_num < end_num: |
212
|
|
|
if len(file_contents[line_num]) > 0 and file_contents[line_num][0] != ' ': |
213
|
|
|
break |
214
|
|
|
line_num += 1 |
215
|
|
|
|
216
|
|
|
end = line_num - 1 |
217
|
|
|
sec_ranges.append((begin, end)) |
218
|
|
|
line_num += 1 |
219
|
|
|
return sec_ranges |
220
|
|
|
|
221
|
|
|
|
222
|
|
|
def remove_lines(file_contents, lines): |
223
|
|
|
# Returns a series of lines and returns a new copy |
224
|
|
|
new_file = [] |
225
|
|
|
for line_num in range(0, len(file_contents)): |
226
|
|
|
if line_num not in lines: |
227
|
|
|
new_file.append(file_contents[line_num]) |
228
|
|
|
|
229
|
|
|
return new_file |
230
|
|
|
|
231
|
|
|
|
232
|
|
|
def remove_section_keys(file_contents, yaml_contents, section, removed_keys): |
233
|
|
|
# Remove a series of keys from a section. Refuses to operate if there is more |
234
|
|
|
# than one instance of the section. If the section is empty (because all keys |
235
|
|
|
# are removed), then the section is also removed. Otherwise, only matching keys |
236
|
|
|
# are removed. Note that all instances of the keys will be removed, if it appears |
237
|
|
|
# more than once. |
238
|
|
|
sec_ranges = find_section_lines(file_contents, section) |
239
|
|
|
if len(sec_ranges) != 1: |
240
|
|
|
raise RuntimeError("Refusing to fix file: %s -- could not find one section: %d" |
241
|
|
|
% (path, sec_ranges)) |
|
|
|
|
242
|
|
|
|
243
|
|
|
begin, end = sec_ranges[0] |
244
|
|
|
r_lines = set() |
245
|
|
|
|
246
|
|
|
if (yaml_contents[section] is None or len(yaml_contents[section].keys()) == len(removed_keys)): |
247
|
|
|
r_lines = set(range(begin, end+1)) |
248
|
|
|
print("Removing entire section since all keys are empty") |
249
|
|
|
else: |
250
|
|
|
# Don't include section header |
251
|
|
|
for line_num in range(begin+1, end+1): |
252
|
|
|
line = file_contents[line_num].strip() |
253
|
|
|
len_line = len(line) |
254
|
|
|
|
255
|
|
|
for key in removed_keys: |
256
|
|
|
k_l = len(key)+1 |
257
|
|
|
k_i = key + ":" |
258
|
|
|
if len_line >= k_l and line[0:k_l] == k_i: |
259
|
|
|
r_lines.add(line_num) |
260
|
|
|
break |
261
|
|
|
|
262
|
|
|
return remove_lines(file_contents, r_lines) |
263
|
|
|
|
264
|
|
|
|
265
|
|
|
def rewrite_value_int_str(line): |
266
|
|
|
# Rewrites a key's value to explicitly be a string. Assumes it starts |
267
|
|
|
# as an integer. Takes a line. |
268
|
|
|
key_end = line.index(':') |
269
|
|
|
key = line[0:key_end] |
270
|
|
|
value = line[key_end+1:].strip() |
271
|
|
|
str_value = '"' + value + '"' |
272
|
|
|
return key + ": " + str_value |
273
|
|
|
|
274
|
|
|
|
275
|
|
|
def rewrite_keyless_section(file_contents, yaml_contents, section, content): |
276
|
|
|
new_contents = file_contents[:] |
277
|
|
|
|
278
|
|
|
sec_ranges = find_section_lines(file_contents, section) |
279
|
|
|
if len(sec_ranges) != 1: |
280
|
|
|
raise RuntimeError("Refusing to fix file: %s -- could not find one section: %d" |
281
|
|
|
% (path, sec_ranges)) |
|
|
|
|
282
|
|
|
|
283
|
|
|
if len(sec_ranges[0]) != 2: |
284
|
|
|
raise RuntimeError("Section has more than one line") |
285
|
|
|
|
286
|
|
|
new_contents[sec_ranges[0][0]] = "{section}: {content}".format(section=section, content=content) |
287
|
|
|
|
288
|
|
|
return new_contents |
289
|
|
|
|
290
|
|
|
|
291
|
|
|
def rewrite_value_remove_prefix(line): |
292
|
|
|
# Rewrites a key's value to remove a "CCE" prefix. |
293
|
|
|
key_end = line.index(':') |
294
|
|
|
key = line[0:key_end] |
295
|
|
|
value = line[key_end+1:].strip() |
296
|
|
|
new_value = value |
297
|
|
|
if cce.is_cce_format_valid("CCE-" + value[3:]): |
298
|
|
|
new_value = value[3:] |
299
|
|
|
elif cce.is_cce_format_valid("CCE-" + value[4:]): |
300
|
|
|
new_value = value[4:] |
301
|
|
|
return key + ": " + new_value |
302
|
|
|
|
303
|
|
|
|
304
|
|
|
def add_to_the_section(file_contents, yaml_contents, section, new_keys): |
305
|
|
|
to_insert = [] |
306
|
|
|
|
307
|
|
|
sec_ranges = find_section_lines(file_contents, section) |
308
|
|
|
if len(sec_ranges) != 1: |
309
|
|
|
raise RuntimeError("could not find one section: %s" |
310
|
|
|
% section) |
311
|
|
|
|
312
|
|
|
begin, end = sec_ranges[0] |
313
|
|
|
|
314
|
|
|
assert end > begin, "We need at least one identifier there already" |
315
|
|
|
template_line = str(file_contents[end - 1]) |
316
|
|
|
leading_whitespace = re.match(r"^\s*", template_line).group() |
317
|
|
|
for key, value in new_keys.items(): |
318
|
|
|
to_insert.append(leading_whitespace + key + ": " + value) |
319
|
|
|
|
320
|
|
|
new_contents = file_contents[:end] + to_insert + file_contents[end:] |
321
|
|
|
return new_contents |
322
|
|
|
|
323
|
|
|
|
324
|
|
|
def sort_section(file_contents, yaml_contents, section): |
325
|
|
|
new_contents = ssg.rule_yaml.sort_section_keys(yaml_contents, file_contents, section) |
326
|
|
|
return new_contents |
327
|
|
|
|
328
|
|
|
|
329
|
|
|
def rewrite_section_value(file_contents, yaml_contents, section, keys, transform): |
330
|
|
|
# For a given section, rewrite the keys in int_keys to be strings. Refuses to |
331
|
|
|
# operate if the given section appears more than once in the file. Assumes all |
332
|
|
|
# instances of key are an integer; all will get updated. |
333
|
|
|
new_contents = file_contents[:] |
334
|
|
|
|
335
|
|
|
sec_ranges = find_section_lines(file_contents, section) |
336
|
|
|
if len(sec_ranges) != 1: |
337
|
|
|
raise RuntimeError("Refusing to fix file: %s -- could not find one section: %d" |
338
|
|
|
% (path, sec_ranges)) |
|
|
|
|
339
|
|
|
|
340
|
|
|
begin, end = sec_ranges[0] |
341
|
|
|
r_lines = set() |
342
|
|
|
|
343
|
|
|
# Don't include section header |
344
|
|
|
for line_num in range(begin+1, end+1): |
345
|
|
|
line = file_contents[line_num].strip() |
346
|
|
|
len_line = len(line) |
347
|
|
|
|
348
|
|
|
for key in keys: |
349
|
|
|
k_l = len(key)+1 |
350
|
|
|
k_i = key + ":" |
351
|
|
|
|
352
|
|
|
if len_line >= k_l and line[0:k_l] == k_i: |
353
|
|
|
new_contents[line_num] = transform(file_contents[line_num]) |
354
|
|
|
break |
355
|
|
|
|
356
|
|
|
return new_contents |
357
|
|
|
|
358
|
|
|
|
359
|
|
|
def rewrite_section_value_int_str(file_contents, yaml_contents, section, int_keys): |
360
|
|
|
return rewrite_section_value(file_contents, yaml_contents, section, int_keys, |
361
|
|
|
rewrite_value_int_str) |
362
|
|
|
|
363
|
|
|
|
364
|
|
View Code Duplication |
def fix_empty_identifier(file_contents, yaml_contents): |
|
|
|
|
365
|
|
|
section = 'identifiers' |
366
|
|
|
|
367
|
|
|
empty_identifiers = [] |
368
|
|
|
if yaml_contents[section] is not None: |
369
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
370
|
|
|
if str(i_value).strip() == "": |
371
|
|
|
empty_identifiers.append(i_type) |
372
|
|
|
|
373
|
|
|
return remove_section_keys(file_contents, yaml_contents, section, empty_identifiers) |
374
|
|
|
|
375
|
|
|
|
376
|
|
View Code Duplication |
def fix_empty_reference(file_contents, yaml_contents): |
|
|
|
|
377
|
|
|
section = 'references' |
378
|
|
|
|
379
|
|
|
empty_identifiers = [] |
380
|
|
|
|
381
|
|
|
if yaml_contents[section] is not None: |
382
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
383
|
|
|
if str(i_value).strip() == "": |
384
|
|
|
empty_identifiers.append(i_type) |
385
|
|
|
|
386
|
|
|
return remove_section_keys(file_contents, yaml_contents, section, empty_identifiers) |
387
|
|
|
|
388
|
|
|
|
389
|
|
|
def fix_prefix_cce(file_contents, yaml_contents): |
390
|
|
|
section = 'identifiers' |
391
|
|
|
|
392
|
|
|
prefixed_identifiers = [] |
393
|
|
|
|
394
|
|
|
if yaml_contents[section] is not None: |
395
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
396
|
|
|
if i_type[0:3] == 'cce': |
397
|
|
|
has_prefix = i_value[0:3].upper() == 'CCE' |
398
|
|
|
remainder_valid = cce.is_cce_format_valid("CCE-" + str(i_value[3:])) |
399
|
|
|
remainder_valid |= cce.is_cce_format_valid("CCE-" + str(i_value[4:])) |
400
|
|
|
if has_prefix and remainder_valid: |
401
|
|
|
prefixed_identifiers.append(i_type) |
402
|
|
|
|
403
|
|
|
return rewrite_section_value(file_contents, yaml_contents, section, prefixed_identifiers, |
404
|
|
|
rewrite_value_remove_prefix) |
405
|
|
|
|
406
|
|
|
|
407
|
|
|
def fix_invalid_cce(file_contents, yaml_contents): |
408
|
|
|
section = 'identifiers' |
409
|
|
|
|
410
|
|
|
invalid_identifiers = [] |
411
|
|
|
|
412
|
|
|
if yaml_contents[section] is not None: |
413
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
414
|
|
|
if i_type[0:3] == 'cce': |
415
|
|
|
if not cce.is_cce_value_valid("CCE-" + str(i_value)): |
416
|
|
|
invalid_identifiers.append(i_type) |
417
|
|
|
|
418
|
|
|
return remove_section_keys(file_contents, yaml_contents, section, invalid_identifiers) |
419
|
|
|
|
420
|
|
|
|
421
|
|
|
def fix_prodtypes(file_contents, yaml_contents): |
422
|
|
|
section = 'prodtype' |
423
|
|
|
sorted_prodtypes = yaml_contents[section].split(",") |
424
|
|
|
sorted_prodtypes.sort() |
425
|
|
|
out = ",".join(sorted_prodtypes) |
426
|
|
|
|
427
|
|
|
return rewrite_keyless_section(file_contents, yaml_contents, section, out) |
428
|
|
|
|
429
|
|
|
|
430
|
|
|
def has_product_cce(yaml_contents, product): |
431
|
|
|
section = 'identifiers' |
432
|
|
|
|
433
|
|
|
invalid_identifiers = [] |
434
|
|
|
|
435
|
|
|
if not yaml_contents[section]: |
436
|
|
|
return False |
437
|
|
|
|
438
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
439
|
|
|
if i_type[0:3] != 'cce' or "@" not in i_type: |
440
|
|
|
continue |
441
|
|
|
|
442
|
|
|
_, cce_product = i_type.split("@", 1) |
443
|
|
|
if product == cce_product: |
444
|
|
|
return True |
445
|
|
|
|
446
|
|
|
return False |
447
|
|
|
|
448
|
|
|
|
449
|
|
|
def add_product_cce(file_contents, yaml_contents, product, cce): |
450
|
|
|
section = 'identifiers' |
451
|
|
|
|
452
|
|
|
if section not in yaml_contents: |
453
|
|
|
return file_contents |
454
|
|
|
|
455
|
|
|
new_contents = add_to_the_section( |
456
|
|
|
file_contents, yaml_contents, section, {"cce@{product}".format(product=product): cce}) |
457
|
|
|
new_contents = sort_section(new_contents, yaml_contents, section) |
458
|
|
|
return new_contents |
459
|
|
|
|
460
|
|
|
|
461
|
|
|
def fix_int_identifier(file_contents, yaml_contents): |
462
|
|
|
section = 'identifiers' |
463
|
|
|
|
464
|
|
|
int_identifiers = [] |
465
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
466
|
|
|
if type(i_value) != str: |
467
|
|
|
int_identifiers.append(i_type) |
468
|
|
|
|
469
|
|
|
return rewrite_section_value_int_str(file_contents, yaml_contents, section, int_identifiers) |
470
|
|
|
|
471
|
|
|
|
472
|
|
|
def fix_int_reference(file_contents, yaml_contents): |
473
|
|
|
section = 'references' |
474
|
|
|
|
475
|
|
|
int_identifiers = [] |
476
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
477
|
|
|
if type(i_value) != str: |
478
|
|
|
int_identifiers.append(i_type) |
479
|
|
|
|
480
|
|
|
return rewrite_section_value_int_str(file_contents, yaml_contents, section, int_identifiers) |
481
|
|
|
|
482
|
|
|
|
483
|
|
|
def sort_rule_subkeys(file_contents, yaml_contents): |
484
|
|
|
return ssg.rule_yaml.sort_section_keys(None, file_contents, TO_SORT) |
485
|
|
|
|
486
|
|
|
|
487
|
|
|
def _fixed_file_contents(path, file_contents, product_yaml, func): |
488
|
|
|
if file_contents[-1] == '': |
489
|
|
|
file_contents = file_contents[:-1] |
490
|
|
|
|
491
|
|
|
subst_dict = product_yaml |
492
|
|
|
yaml_contents = yaml.open_and_macro_expand(path, subst_dict) |
493
|
|
|
|
494
|
|
|
try: |
495
|
|
|
new_file_contents = func(file_contents, yaml_contents) |
496
|
|
|
except Exception as exc: |
497
|
|
|
msg = "Refusing to fix file: {path}: {error}".format(path=path, error=str(exc)) |
498
|
|
|
raise RuntimeError(msg) |
499
|
|
|
|
500
|
|
|
return new_file_contents |
501
|
|
|
|
502
|
|
|
|
503
|
|
|
def fix_file(path, product_yaml, func): |
504
|
|
|
file_contents = open(path, 'r').read().split("\n") |
505
|
|
|
|
506
|
|
|
new_file_contents = _fixed_file_contents(path, file_contents, product_yaml, func) |
507
|
|
|
if file_contents == new_file_contents: |
508
|
|
|
return False |
509
|
|
|
|
510
|
|
|
with open(path, 'w') as f: |
511
|
|
|
for line in new_file_contents: |
512
|
|
|
print(line, file=f) |
513
|
|
|
return True |
514
|
|
|
|
515
|
|
|
|
516
|
|
|
def fix_file_prompt(path, product_yaml, func, args): |
517
|
|
|
file_contents = open(path, 'r').read().split("\n") |
518
|
|
|
|
519
|
|
|
new_file_contents = _fixed_file_contents(path, file_contents, product_yaml, func) |
520
|
|
|
changes = file_contents != new_file_contents |
521
|
|
|
|
522
|
|
|
if not changes: |
523
|
|
|
return changes |
524
|
|
|
|
525
|
|
|
need_input = not args.assume_yes and not args.dry_run |
526
|
|
|
|
527
|
|
|
if need_input: |
528
|
|
|
print("====BEGIN BEFORE====") |
529
|
|
|
print_file(file_contents) |
530
|
|
|
print("====END BEFORE====") |
531
|
|
|
|
532
|
|
|
if need_input: |
533
|
|
|
print("====BEGIN AFTER====") |
534
|
|
|
print_file(new_file_contents) |
535
|
|
|
print("====END AFTER====") |
536
|
|
|
|
537
|
|
|
response = 'n' |
538
|
|
|
if need_input: |
539
|
|
|
response = input_func("Confirm writing output to %s: (y/n): " % path) |
540
|
|
|
|
541
|
|
|
if args.assume_yes or response.strip().lower() == 'y': |
542
|
|
|
changes = True |
543
|
|
|
with open(path, 'w') as f: |
544
|
|
|
for line in new_file_contents: |
545
|
|
|
print(line, file=f) |
546
|
|
|
else: |
547
|
|
|
changes = False |
548
|
|
|
return changes |
549
|
|
|
|
550
|
|
|
|
551
|
|
|
def add_cce(args, product_yaml): |
552
|
|
|
directory = os.path.join(args.root, args.subdirectory) |
553
|
|
|
cce_pool = cce.CCE_POOLS[args.cce_pool]() |
554
|
|
|
return _add_cce(directory, cce_pool, args.rule, product_yaml, args) |
555
|
|
|
|
556
|
|
|
|
557
|
|
|
def _add_cce(directory, cce_pool, rules, product_yaml, args): |
558
|
|
|
product = product_yaml["product"] |
559
|
|
|
|
560
|
|
|
def is_relevant_rule(rule_path, rule, rule_lines): |
561
|
|
|
for r in rules: |
562
|
|
|
if ( |
563
|
|
|
rule_path.endswith("/{r}/rule.yml".format(r=r)) |
564
|
|
|
and has_no_cce(rule_path, product_yaml)): |
565
|
|
|
return True |
566
|
|
|
return False |
567
|
|
|
|
568
|
|
|
results = find_rules(args, is_relevant_rule) |
569
|
|
|
|
570
|
|
|
for result in results: |
571
|
|
|
rule_path = result[0] |
572
|
|
|
|
573
|
|
|
cce = cce_pool.random_cce() |
574
|
|
|
|
575
|
|
|
def fix_callback(file_contents, yaml_contents): |
576
|
|
|
return add_product_cce(file_contents, yaml_contents, product_yaml["product"], cce) |
|
|
|
|
577
|
|
|
|
578
|
|
|
try: |
579
|
|
|
changes = fix_file(rule_path, product_yaml, fix_callback) |
580
|
|
|
except RuntimeError as exc: |
581
|
|
|
msg = ( |
582
|
|
|
"Error adding CCE into {rule_path}: {exc}" |
583
|
|
|
.format(rule_path=rule_path, exc=str(exc))) |
584
|
|
|
raise RuntimeError(exc) |
585
|
|
|
|
586
|
|
|
if changes: |
587
|
|
|
cce_pool.remove_cce_from_file(cce) |
588
|
|
|
|
589
|
|
|
|
590
|
|
|
def has_unsorted_prodtype(rule_path, rule, rule_lines): |
591
|
|
|
if 'prodtype' in rule: |
592
|
|
|
prodtypes = rule['prodtype'].split(',') |
593
|
|
|
return prodtypes != sorted(prodtypes) |
594
|
|
|
return False |
595
|
|
|
|
596
|
|
|
|
597
|
|
|
@command("empty_identifiers", "check and fix rules with empty identifiers") |
598
|
|
|
def fix_empty_identifiers(args, product_yaml): |
599
|
|
|
results = find_rules(args, has_empty_identifier) |
600
|
|
|
print("Number of rules with empty identifiers: %d" % len(results)) |
601
|
|
|
|
602
|
|
|
for result in results: |
603
|
|
|
rule_path = result[0] |
604
|
|
|
|
605
|
|
|
product_yaml_path = result[2] |
606
|
|
|
|
607
|
|
|
if product_yaml_path is not None: |
608
|
|
|
product_yaml = yaml.open_raw(product_yaml_path) |
609
|
|
|
|
610
|
|
|
if args.dry_run: |
611
|
|
|
print(rule_path + " has one or more empty identifiers") |
612
|
|
|
continue |
613
|
|
|
|
614
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_empty_identifier, args) |
615
|
|
|
|
616
|
|
|
exit(int(len(results) > 0)) |
617
|
|
|
|
618
|
|
|
|
619
|
|
|
@command("empty_references", "check and fix rules with empty references") |
620
|
|
|
def fix_empty_references(args, product_yaml): |
621
|
|
|
results = find_rules(args, has_empty_references) |
622
|
|
|
print("Number of rules with empty references: %d" % len(results)) |
623
|
|
|
|
624
|
|
|
for result in results: |
625
|
|
|
rule_path = result[0] |
626
|
|
|
product_yaml = result[2] |
627
|
|
|
|
628
|
|
|
if args.dry_run: |
629
|
|
|
print(rule_path + " has one or more empty references") |
630
|
|
|
continue |
631
|
|
|
|
632
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_empty_reference, args) |
633
|
|
|
|
634
|
|
|
exit(int(len(results) > 0)) |
635
|
|
|
|
636
|
|
|
|
637
|
|
|
@command("prefixed_identifiers", "check and fix rules with prefixed (CCE-) identifiers") |
638
|
|
|
def find_prefix_cce(args): |
639
|
|
|
results = find_rules(args, has_prefix_cce) |
640
|
|
|
print("Number of rules with prefixed CCEs: %d" % len(results)) |
641
|
|
|
|
642
|
|
|
for result in results: |
643
|
|
|
rule_path = result[0] |
644
|
|
|
product_yaml = result[2] |
645
|
|
|
|
646
|
|
|
if args.dry_run: |
647
|
|
|
print(rule_path + " has one or more CCE with CCE- prefix") |
648
|
|
|
continue |
649
|
|
|
|
650
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_prefix_cce, args) |
651
|
|
|
|
652
|
|
|
exit(int(len(results) > 0)) |
653
|
|
|
|
654
|
|
|
|
655
|
|
|
@command("invalid_identifiers", "check and fix rules with invalid identifiers") |
656
|
|
|
def find_invalid_cce(args, product_yamls): |
657
|
|
|
results = find_rules(args, has_invalid_cce) |
658
|
|
|
print("Number of rules with invalid CCEs: %d" % len(results)) |
659
|
|
|
|
660
|
|
|
for result in results: |
661
|
|
|
rule_path = result[0] |
662
|
|
|
product_yaml = result[2] |
663
|
|
|
|
664
|
|
|
if args.dry_run: |
665
|
|
|
print(rule_path + " has one or more invalid CCEs") |
666
|
|
|
continue |
667
|
|
|
|
668
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_invalid_cce, args) |
669
|
|
|
exit(int(len(results) > 0)) |
670
|
|
|
|
671
|
|
|
|
672
|
|
|
@command("int_identifiers", "check and fix rules with pseudo-integer identifiers") |
673
|
|
|
def find_int_identifiers(args, product_yaml): |
674
|
|
|
results = find_rules(args, has_int_identifier) |
675
|
|
|
print("Number of rules with integer identifiers: %d" % len(results)) |
676
|
|
|
|
677
|
|
|
for result in results: |
678
|
|
|
rule_path = result[0] |
679
|
|
|
product_yaml = result[2] |
680
|
|
|
|
681
|
|
|
if args.dry_run: |
682
|
|
|
print(rule_path + " has one or more integer references") |
683
|
|
|
continue |
684
|
|
|
|
685
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_int_identifier, args) |
686
|
|
|
|
687
|
|
|
exit(int(len(results) > 0)) |
688
|
|
|
|
689
|
|
|
|
690
|
|
|
@command("int_references", "check and fix rules with pseudo-integer references") |
691
|
|
|
def find_int_references(args, product_yaml): |
692
|
|
|
results = find_rules(args, has_int_reference) |
693
|
|
|
print("Number of rules with integer references: %d" % len(results)) |
694
|
|
|
|
695
|
|
|
for result in results: |
696
|
|
|
rule_path = result[0] |
697
|
|
|
product_yaml = result[2] |
698
|
|
|
|
699
|
|
|
if args.dry_run: |
700
|
|
|
print(rule_path + " has one or more unsorted references") |
701
|
|
|
continue |
702
|
|
|
|
703
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_int_reference, args) |
704
|
|
|
|
705
|
|
|
exit(int(len(results) > 0)) |
706
|
|
|
|
707
|
|
|
|
708
|
|
|
@command("duplicate_subkeys", "check for duplicated references and identifiers") |
709
|
|
|
def duplicate_subkeys(args, product_yaml): |
710
|
|
|
results = find_rules(args, has_duplicated_subkeys) |
711
|
|
|
print("Number of rules with duplicated subkeys: %d" % len(results)) |
712
|
|
|
|
713
|
|
|
for result in results: |
714
|
|
|
print(result[0] + " has one or more duplicated subkeys") |
715
|
|
|
|
716
|
|
|
exit(int(len(results) > 0)) |
717
|
|
|
|
718
|
|
|
|
719
|
|
|
@command("sort_subkeys", "sort references and identifiers") |
720
|
|
|
def sort_subkeys(args, product_yaml): |
721
|
|
|
results = find_rules(args, has_unordered_sections) |
722
|
|
|
print("Number of modified rules: %d" % len(results)) |
723
|
|
|
|
724
|
|
|
for result in results: |
725
|
|
|
rule_path = result[0] |
726
|
|
|
product_yaml = result[2] |
727
|
|
|
|
728
|
|
|
if args.dry_run: |
729
|
|
|
print(rule_path + " has one or more unsorted references") |
730
|
|
|
continue |
731
|
|
|
|
732
|
|
|
fix_file_prompt(rule_path, product_yaml, sort_rule_subkeys, args) |
733
|
|
|
|
734
|
|
|
exit(int(len(results) > 0)) |
735
|
|
|
|
736
|
|
|
|
737
|
|
|
@command("sort_prodtypes", "sorts the products in the prodtype") |
738
|
|
|
def sort_prodtypes(args, product_yaml): |
739
|
|
|
results = find_rules(args, has_unsorted_prodtype) |
740
|
|
|
for result in results: |
741
|
|
|
rule_path = result[0] |
742
|
|
|
product_yaml = result[2] |
743
|
|
|
|
744
|
|
|
if args.dry_run: |
745
|
|
|
print(rule_path + " prodtype is unsorted") |
746
|
|
|
continue |
747
|
|
|
|
748
|
|
|
fix_file(rule_path, product_yaml, fix_prodtypes) |
749
|
|
|
|
750
|
|
|
exit(int(len(results) > 0)) |
751
|
|
|
|
752
|
|
|
|
753
|
|
|
@command("test_all", "Perform all checks on all rules") |
754
|
|
|
def test_all(args, product_yaml): |
755
|
|
|
result = 0 |
756
|
|
|
checks = [ |
757
|
|
|
(has_empty_identifier, "empty identifiers"), |
758
|
|
|
(has_invalid_cce, "invalid CCEs"), |
759
|
|
|
(has_int_identifier, "integer references"), |
760
|
|
|
(has_empty_references, "empty references"), |
761
|
|
|
(has_int_reference, "unsorted references"), |
762
|
|
|
(has_duplicated_subkeys, "duplicated subkeys"), |
763
|
|
|
(has_unordered_sections, "unsorted references"), |
764
|
|
|
(has_unsorted_prodtype, "unsorted prodtype") |
765
|
|
|
] |
766
|
|
|
for item in rule_data_generator(args): |
767
|
|
|
rule_path, rule, rule_lines, _, _ = item |
768
|
|
|
for func, msg in checks: |
769
|
|
|
if func(rule_path, rule, rule_lines): |
770
|
|
|
print("Rule '%s' has %s" % (rule_path, msg)) |
771
|
|
|
result = 1 |
772
|
|
|
exit(result) |
773
|
|
|
|
774
|
|
|
|
775
|
|
|
def create_parser_from_functions(subparsers): |
776
|
|
|
for name, function in _COMMANDS.items(): |
777
|
|
|
subparser = subparsers.add_parser(name, description=function.description) |
778
|
|
|
subparser.set_defaults(func=function) |
779
|
|
|
|
780
|
|
|
|
781
|
|
|
def create_other_parsers(subparsers): |
782
|
|
|
subparser = subparsers.add_parser("add-cce", description="Add CCE to rule files") |
783
|
|
|
subparser.add_argument("rule", nargs="+") |
784
|
|
|
subparser.add_argument("--subdirectory", default="linux_os") |
785
|
|
|
subparser.add_argument( |
786
|
|
|
"--cce-pool", "-p", default="redhat", choices=list(cce.CCE_POOLS.keys()), |
787
|
|
|
) |
788
|
|
|
subparser.set_defaults(func=add_cce) |
789
|
|
|
|
790
|
|
|
|
791
|
|
|
def parse_args(): |
792
|
|
|
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, |
793
|
|
|
description="Utility for fixing mistakes in rule files") |
794
|
|
|
parser.add_argument( |
795
|
|
|
"-y", "--assume-yes", default=False, action="store_true", |
796
|
|
|
help="Assume yes and overwrite all files (no prompt)") |
797
|
|
|
parser.add_argument( |
798
|
|
|
"-d", "--dry-run", default=False, action="store_true", |
799
|
|
|
help="Assume no and don't overwrite any files") |
800
|
|
|
parser.add_argument( |
801
|
|
|
"-j", "--json", type=str, action="store", |
802
|
|
|
default="build/rule_dirs.json", help="File to read json " |
803
|
|
|
"output of rule_dir_json.py from (defaults to " |
804
|
|
|
"build/rule_dirs.json") |
805
|
|
|
parser.add_argument( |
806
|
|
|
"-r", "--root", default=SSG_ROOT, |
807
|
|
|
help="Path to root of the project directory") |
808
|
|
|
parser.add_argument("--product", "-p", help="Path to the main product.yml") |
809
|
|
|
subparsers = parser.add_subparsers(title="command", help="What to perform.") |
810
|
|
|
subparsers.required = True |
811
|
|
|
create_parser_from_functions(subparsers) |
812
|
|
|
create_other_parsers(subparsers) |
813
|
|
|
return parser.parse_args() |
814
|
|
|
|
815
|
|
|
|
816
|
|
|
def __main__(): |
817
|
|
|
args = parse_args() |
818
|
|
|
project_root = args.root |
819
|
|
|
if not project_root: |
820
|
|
|
project_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.pardir) |
821
|
|
|
|
822
|
|
|
subst_dict = dict() |
823
|
|
|
if args.product: |
824
|
|
|
subst_dict = dict() |
825
|
|
|
product = products.load_product_yaml(args.product) |
826
|
|
|
product.read_properties_from_directory(os.path.join(project_root, "product_properties")) |
827
|
|
|
subst_dict.update(product) |
828
|
|
|
|
829
|
|
|
args.func(args, subst_dict) |
830
|
|
|
|
831
|
|
|
|
832
|
|
|
if __name__ == "__main__": |
833
|
|
|
__main__() |
834
|
|
|
|