|
1
|
|
|
#!/usr/bin/python3 |
|
2
|
|
|
|
|
3
|
|
|
from __future__ import print_function |
|
4
|
|
|
|
|
5
|
|
|
import sys |
|
6
|
|
|
import os |
|
7
|
|
|
import jinja2 |
|
8
|
|
|
import argparse |
|
9
|
|
|
import json |
|
10
|
|
|
import re |
|
11
|
|
|
import random |
|
12
|
|
|
|
|
13
|
|
|
from ssg import yaml, cce, products |
|
14
|
|
|
from ssg.shims import input_func |
|
15
|
|
|
from ssg.utils import read_file_list |
|
16
|
|
|
import ssg |
|
17
|
|
|
import ssg.products |
|
18
|
|
|
import ssg.rules |
|
19
|
|
|
import ssg.rule_yaml |
|
20
|
|
|
|
|
21
|
|
|
|
|
22
|
|
|
SSG_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) |
|
23
|
|
|
TO_SORT = ['identifiers', 'references'] |
|
24
|
|
|
|
|
25
|
|
|
|
|
26
|
|
|
_COMMANDS = dict() |
|
27
|
|
|
|
|
28
|
|
|
|
|
29
|
|
|
def command(name, description): |
|
30
|
|
|
def wrapper(wrapped): |
|
31
|
|
|
_COMMANDS[name] = wrapped |
|
32
|
|
|
wrapped.description = description |
|
33
|
|
|
return wrapped |
|
34
|
|
|
return wrapper |
|
35
|
|
|
|
|
36
|
|
|
|
|
37
|
|
|
def has_empty_identifier(rule_path, rule, rule_lines): |
|
38
|
|
|
if 'identifiers' in rule and rule['identifiers'] is None: |
|
39
|
|
|
return True |
|
40
|
|
|
|
|
41
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
|
42
|
|
|
for _, value in rule['identifiers'].items(): |
|
43
|
|
|
if str(value).strip() == "": |
|
44
|
|
|
return True |
|
45
|
|
|
return False |
|
46
|
|
|
|
|
47
|
|
|
|
|
48
|
|
|
def has_no_cce(yaml_file, product_yaml=None): |
|
49
|
|
|
rule = yaml.open_and_macro_expand(yaml_file, product_yaml) |
|
50
|
|
|
product = product_yaml["product"] |
|
51
|
|
|
if "prodtype" in rule and product not in rule["prodtype"]: |
|
52
|
|
|
return False |
|
53
|
|
|
if 'identifiers' in rule and rule['identifiers'] is None: |
|
54
|
|
|
return True |
|
55
|
|
|
|
|
56
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
|
57
|
|
|
for ident in rule['identifiers']: |
|
58
|
|
|
if ident == "cce@" + product: |
|
59
|
|
|
return False |
|
60
|
|
|
return True |
|
61
|
|
|
|
|
62
|
|
|
|
|
63
|
|
|
def has_empty_references(rule_path, rule, rule_lines): |
|
64
|
|
|
if 'references' in rule and rule['references'] is None: |
|
65
|
|
|
return True |
|
66
|
|
|
|
|
67
|
|
|
if 'references' in rule and rule['references'] is not None: |
|
68
|
|
|
for _, value in rule['references'].items(): |
|
69
|
|
|
if str(value).strip() == "": |
|
70
|
|
|
return True |
|
71
|
|
|
return False |
|
72
|
|
|
|
|
73
|
|
|
|
|
74
|
|
|
def has_prefix_cce(rule_path, rule, rule_lines): |
|
75
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
|
76
|
|
|
for i_type, i_value in rule['identifiers'].items(): |
|
77
|
|
|
if i_type[0:3] == 'cce': |
|
78
|
|
|
has_prefix = i_value[0:3].upper() == 'CCE' |
|
79
|
|
|
remainder_valid = cce.is_cce_format_valid("CCE-" + i_value[3:]) |
|
80
|
|
|
remainder_valid |= cce.is_cce_format_valid("CCE-" + i_value[4:]) |
|
81
|
|
|
return has_prefix and remainder_valid |
|
82
|
|
|
return False |
|
83
|
|
|
|
|
84
|
|
|
|
|
85
|
|
|
def has_invalid_cce(rule_path, rule, rule_lines): |
|
86
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
|
87
|
|
|
for i_type, i_value in rule['identifiers'].items(): |
|
88
|
|
|
if i_type[0:3] == 'cce': |
|
89
|
|
|
if not cce.is_cce_value_valid("CCE-" + str(i_value)): |
|
90
|
|
|
return True |
|
91
|
|
|
return False |
|
92
|
|
|
|
|
93
|
|
|
|
|
94
|
|
|
def has_int_identifier(rule_path, rule, rule_lines): |
|
95
|
|
|
if 'identifiers' in rule and rule['identifiers'] is not None: |
|
96
|
|
|
for _, value in rule['identifiers'].items(): |
|
97
|
|
|
if type(value) != str: |
|
98
|
|
|
return True |
|
99
|
|
|
return False |
|
100
|
|
|
|
|
101
|
|
|
|
|
102
|
|
|
def has_int_reference(rule_path, rule, rule_lines): |
|
103
|
|
|
if 'references' in rule and rule['references'] is not None: |
|
104
|
|
|
for _, value in rule['references'].items(): |
|
105
|
|
|
if type(value) != str: |
|
106
|
|
|
return True |
|
107
|
|
|
return False |
|
108
|
|
|
|
|
109
|
|
|
|
|
110
|
|
|
def has_duplicated_subkeys(rule_path, rule, rule_lines): |
|
111
|
|
|
return ssg.rule_yaml.has_duplicated_subkeys(rule_path, rule_lines, TO_SORT) |
|
112
|
|
|
|
|
113
|
|
|
|
|
114
|
|
|
def has_unordered_sections(rule_path, rule, rule_lines): |
|
115
|
|
|
if 'references' in rule or 'identifiers' in rule: |
|
116
|
|
|
new_lines = ssg.rule_yaml.sort_section_keys(rule_path, rule_lines, TO_SORT) |
|
117
|
|
|
|
|
118
|
|
|
# Compare string representations to avoid issues with references being |
|
119
|
|
|
# different. |
|
120
|
|
|
return "\n".join(rule_lines) != "\n".join(new_lines) |
|
121
|
|
|
|
|
122
|
|
|
return False |
|
123
|
|
|
|
|
124
|
|
|
|
|
125
|
|
|
def rule_data_generator(args): |
|
126
|
|
|
# Iterates over all know rules in the build system (according to |
|
127
|
|
|
# rule_dir_json.py) and attempts to load the resulting YAML files. |
|
128
|
|
|
# If they parse correctly, yield them as a result. |
|
129
|
|
|
# |
|
130
|
|
|
# Note: this has become a generator rather than returning a list of |
|
131
|
|
|
# results. |
|
132
|
|
|
|
|
133
|
|
|
product_yamls = dict() |
|
134
|
|
|
|
|
135
|
|
|
rule_dirs = json.load(open(args.json)) |
|
136
|
|
|
for rule_id in rule_dirs: |
|
137
|
|
|
rule_obj = rule_dirs[rule_id] |
|
138
|
|
|
|
|
139
|
|
|
if 'products' not in rule_obj or not rule_obj['products']: |
|
140
|
|
|
print(rule_id, rule_obj) |
|
141
|
|
|
assert rule_obj['products'] |
|
142
|
|
|
product = rule_obj['products'][0] |
|
143
|
|
|
|
|
144
|
|
|
if product not in product_yamls: |
|
145
|
|
|
product_path = ssg.products.product_yaml_path(args.root, product) |
|
146
|
|
|
product_yaml = ssg.products.load_product_yaml(product_path) |
|
147
|
|
|
properties_directory = os.path.join(args.root, "product_properties") |
|
148
|
|
|
product_yaml.read_properties_from_directory(properties_directory) |
|
149
|
|
|
product_yamls[product] = product_yaml |
|
150
|
|
|
|
|
151
|
|
|
local_env_yaml = dict(cmake_build_type='Debug') |
|
152
|
|
|
local_env_yaml.update(product_yamls[product]) |
|
153
|
|
|
local_env_yaml['rule_id'] = rule_id |
|
154
|
|
|
|
|
155
|
|
|
rule_path = ssg.rules.get_rule_dir_yaml(rule_obj['dir']) |
|
156
|
|
|
try: |
|
157
|
|
|
rule = yaml.open_and_macro_expand(rule_path, local_env_yaml) |
|
158
|
|
|
rule_lines = read_file_list(rule_path) |
|
159
|
|
|
yield rule_path, rule, rule_lines, product_path, local_env_yaml |
|
|
|
|
|
|
160
|
|
|
except jinja2.exceptions.UndefinedError as ue: |
|
161
|
|
|
msg = "Failed to parse file {0} (with product.yml: {1}). Skipping. {2}" |
|
162
|
|
|
msg = msg.format(rule_path, product_path, ue) |
|
163
|
|
|
print(msg, file=sys.stderr) |
|
164
|
|
|
|
|
165
|
|
|
|
|
166
|
|
|
def find_rules_generator(args, func): |
|
167
|
|
|
for item in rule_data_generator(args): |
|
168
|
|
|
rule_path, rule, rule_lines, product_path, local_env_yaml = item |
|
169
|
|
|
if func(rule_path, rule, rule_lines): |
|
170
|
|
|
yield (rule_path, product_path, local_env_yaml) |
|
171
|
|
|
|
|
172
|
|
|
|
|
173
|
|
|
def find_rules(args, func): |
|
174
|
|
|
# Returns find_rules_generator as a list |
|
175
|
|
|
return list(find_rules_generator(args, func)) |
|
176
|
|
|
|
|
177
|
|
|
|
|
178
|
|
|
def print_file(file_contents): |
|
179
|
|
|
for line_num in range(0, len(file_contents)): |
|
180
|
|
|
print("%d: %s" % (line_num, file_contents[line_num])) |
|
181
|
|
|
|
|
182
|
|
|
|
|
183
|
|
|
def find_section_lines(file_contents, sec): |
|
184
|
|
|
# Hack to find a global key ("section"/sec) in a YAML-like file. |
|
185
|
|
|
# All indented lines until the next global key are included in the range. |
|
186
|
|
|
# For example: |
|
187
|
|
|
# |
|
188
|
|
|
# 0: not_it: |
|
189
|
|
|
# 1: - value |
|
190
|
|
|
# 2: this_one: |
|
191
|
|
|
# 3: - 2 |
|
192
|
|
|
# 4: - 5 |
|
193
|
|
|
# 5: |
|
194
|
|
|
# 6: nor_this: |
|
195
|
|
|
# |
|
196
|
|
|
# for the section "this_one", the result [(2, 5)] will be returned. |
|
197
|
|
|
# Note that multiple sections may exist in a file and each will be |
|
198
|
|
|
# identified and returned. |
|
199
|
|
|
sec_ranges = [] |
|
200
|
|
|
|
|
201
|
|
|
sec_id = sec + ":" |
|
202
|
|
|
sec_len = len(sec_id) |
|
203
|
|
|
end_num = len(file_contents) |
|
204
|
|
|
line_num = 0 |
|
205
|
|
|
|
|
206
|
|
View Code Duplication |
while line_num < end_num: |
|
|
|
|
|
|
207
|
|
|
if len(file_contents[line_num]) >= sec_len: |
|
208
|
|
|
if file_contents[line_num][0:sec_len] == sec_id: |
|
209
|
|
|
begin = line_num |
|
210
|
|
|
line_num += 1 |
|
211
|
|
|
while line_num < end_num: |
|
212
|
|
|
if len(file_contents[line_num]) > 0 and file_contents[line_num][0] != ' ': |
|
213
|
|
|
break |
|
214
|
|
|
line_num += 1 |
|
215
|
|
|
|
|
216
|
|
|
end = line_num - 1 |
|
217
|
|
|
sec_ranges.append((begin, end)) |
|
218
|
|
|
line_num += 1 |
|
219
|
|
|
return sec_ranges |
|
220
|
|
|
|
|
221
|
|
|
|
|
222
|
|
|
def remove_lines(file_contents, lines): |
|
223
|
|
|
# Returns a series of lines and returns a new copy |
|
224
|
|
|
new_file = [] |
|
225
|
|
|
for line_num in range(0, len(file_contents)): |
|
226
|
|
|
if line_num not in lines: |
|
227
|
|
|
new_file.append(file_contents[line_num]) |
|
228
|
|
|
|
|
229
|
|
|
return new_file |
|
230
|
|
|
|
|
231
|
|
|
|
|
232
|
|
|
def remove_section_keys(file_contents, yaml_contents, section, removed_keys): |
|
233
|
|
|
# Remove a series of keys from a section. Refuses to operate if there is more |
|
234
|
|
|
# than one instance of the section. If the section is empty (because all keys |
|
235
|
|
|
# are removed), then the section is also removed. Otherwise, only matching keys |
|
236
|
|
|
# are removed. Note that all instances of the keys will be removed, if it appears |
|
237
|
|
|
# more than once. |
|
238
|
|
|
sec_ranges = find_section_lines(file_contents, section) |
|
239
|
|
|
if len(sec_ranges) != 1: |
|
240
|
|
|
raise RuntimeError("Refusing to fix file: %s -- could not find one section: %d" |
|
241
|
|
|
% (path, sec_ranges)) |
|
|
|
|
|
|
242
|
|
|
|
|
243
|
|
|
begin, end = sec_ranges[0] |
|
244
|
|
|
r_lines = set() |
|
245
|
|
|
|
|
246
|
|
|
if (yaml_contents[section] is None or len(yaml_contents[section].keys()) == len(removed_keys)): |
|
247
|
|
|
r_lines = set(range(begin, end+1)) |
|
248
|
|
|
print("Removing entire section since all keys are empty") |
|
249
|
|
|
else: |
|
250
|
|
|
# Don't include section header |
|
251
|
|
|
for line_num in range(begin+1, end+1): |
|
252
|
|
|
line = file_contents[line_num].strip() |
|
253
|
|
|
len_line = len(line) |
|
254
|
|
|
|
|
255
|
|
|
for key in removed_keys: |
|
256
|
|
|
k_l = len(key)+1 |
|
257
|
|
|
k_i = key + ":" |
|
258
|
|
|
if len_line >= k_l and line[0:k_l] == k_i: |
|
259
|
|
|
r_lines.add(line_num) |
|
260
|
|
|
break |
|
261
|
|
|
|
|
262
|
|
|
return remove_lines(file_contents, r_lines) |
|
263
|
|
|
|
|
264
|
|
|
|
|
265
|
|
|
def rewrite_value_int_str(line): |
|
266
|
|
|
# Rewrites a key's value to explicitly be a string. Assumes it starts |
|
267
|
|
|
# as an integer. Takes a line. |
|
268
|
|
|
key_end = line.index(':') |
|
269
|
|
|
key = line[0:key_end] |
|
270
|
|
|
value = line[key_end+1:].strip() |
|
271
|
|
|
str_value = '"' + value + '"' |
|
272
|
|
|
return key + ": " + str_value |
|
273
|
|
|
|
|
274
|
|
|
|
|
275
|
|
|
def rewrite_keyless_section(file_contents, yaml_contents, section, content): |
|
276
|
|
|
new_contents = file_contents[:] |
|
277
|
|
|
|
|
278
|
|
|
sec_ranges = find_section_lines(file_contents, section) |
|
279
|
|
|
if len(sec_ranges) != 1: |
|
280
|
|
|
raise RuntimeError("Refusing to fix file: %s -- could not find one section: %d" |
|
281
|
|
|
% (path, sec_ranges)) |
|
|
|
|
|
|
282
|
|
|
|
|
283
|
|
|
if len(sec_ranges[0]) != 2: |
|
284
|
|
|
raise RuntimeError("Section has more than one line") |
|
285
|
|
|
|
|
286
|
|
|
new_contents[sec_ranges[0][0]] = "{section}: {content}".format(section=section, content=content) |
|
287
|
|
|
|
|
288
|
|
|
return new_contents |
|
289
|
|
|
|
|
290
|
|
|
|
|
291
|
|
|
def rewrite_value_remove_prefix(line): |
|
292
|
|
|
# Rewrites a key's value to remove a "CCE" prefix. |
|
293
|
|
|
key_end = line.index(':') |
|
294
|
|
|
key = line[0:key_end] |
|
295
|
|
|
value = line[key_end+1:].strip() |
|
296
|
|
|
new_value = value |
|
297
|
|
|
if cce.is_cce_format_valid("CCE-" + value[3:]): |
|
298
|
|
|
new_value = value[3:] |
|
299
|
|
|
elif cce.is_cce_format_valid("CCE-" + value[4:]): |
|
300
|
|
|
new_value = value[4:] |
|
301
|
|
|
return key + ": " + new_value |
|
302
|
|
|
|
|
303
|
|
|
|
|
304
|
|
|
def add_to_the_section(file_contents, yaml_contents, section, new_keys): |
|
305
|
|
|
to_insert = [] |
|
306
|
|
|
|
|
307
|
|
|
sec_ranges = find_section_lines(file_contents, section) |
|
308
|
|
|
if len(sec_ranges) != 1: |
|
309
|
|
|
raise RuntimeError("could not find one section: %s" |
|
310
|
|
|
% section) |
|
311
|
|
|
|
|
312
|
|
|
begin, end = sec_ranges[0] |
|
313
|
|
|
|
|
314
|
|
|
assert end > begin, "We need at least one identifier there already" |
|
315
|
|
|
template_line = str(file_contents[end - 1]) |
|
316
|
|
|
leading_whitespace = re.match(r"^\s*", template_line).group() |
|
317
|
|
|
for key, value in new_keys.items(): |
|
318
|
|
|
to_insert.append(leading_whitespace + key + ": " + value) |
|
319
|
|
|
|
|
320
|
|
|
new_contents = file_contents[:end] + to_insert + file_contents[end:] |
|
321
|
|
|
return new_contents |
|
322
|
|
|
|
|
323
|
|
|
|
|
324
|
|
|
def sort_section(file_contents, yaml_contents, section): |
|
325
|
|
|
new_contents = ssg.rule_yaml.sort_section_keys(yaml_contents, file_contents, section) |
|
326
|
|
|
return new_contents |
|
327
|
|
|
|
|
328
|
|
|
|
|
329
|
|
|
def rewrite_section_value(file_contents, yaml_contents, section, keys, transform): |
|
330
|
|
|
# For a given section, rewrite the keys in int_keys to be strings. Refuses to |
|
331
|
|
|
# operate if the given section appears more than once in the file. Assumes all |
|
332
|
|
|
# instances of key are an integer; all will get updated. |
|
333
|
|
|
new_contents = file_contents[:] |
|
334
|
|
|
|
|
335
|
|
|
sec_ranges = find_section_lines(file_contents, section) |
|
336
|
|
|
if len(sec_ranges) != 1: |
|
337
|
|
|
raise RuntimeError("Refusing to fix file: %s -- could not find one section: %d" |
|
338
|
|
|
% (path, sec_ranges)) |
|
|
|
|
|
|
339
|
|
|
|
|
340
|
|
|
begin, end = sec_ranges[0] |
|
341
|
|
|
r_lines = set() |
|
342
|
|
|
|
|
343
|
|
|
# Don't include section header |
|
344
|
|
|
for line_num in range(begin+1, end+1): |
|
345
|
|
|
line = file_contents[line_num].strip() |
|
346
|
|
|
len_line = len(line) |
|
347
|
|
|
|
|
348
|
|
|
for key in keys: |
|
349
|
|
|
k_l = len(key)+1 |
|
350
|
|
|
k_i = key + ":" |
|
351
|
|
|
|
|
352
|
|
|
if len_line >= k_l and line[0:k_l] == k_i: |
|
353
|
|
|
new_contents[line_num] = transform(file_contents[line_num]) |
|
354
|
|
|
break |
|
355
|
|
|
|
|
356
|
|
|
return new_contents |
|
357
|
|
|
|
|
358
|
|
|
|
|
359
|
|
|
def rewrite_section_value_int_str(file_contents, yaml_contents, section, int_keys): |
|
360
|
|
|
return rewrite_section_value(file_contents, yaml_contents, section, int_keys, |
|
361
|
|
|
rewrite_value_int_str) |
|
362
|
|
|
|
|
363
|
|
|
|
|
364
|
|
View Code Duplication |
def fix_empty_identifier(file_contents, yaml_contents): |
|
|
|
|
|
|
365
|
|
|
section = 'identifiers' |
|
366
|
|
|
|
|
367
|
|
|
empty_identifiers = [] |
|
368
|
|
|
if yaml_contents[section] is not None: |
|
369
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
|
370
|
|
|
if str(i_value).strip() == "": |
|
371
|
|
|
empty_identifiers.append(i_type) |
|
372
|
|
|
|
|
373
|
|
|
return remove_section_keys(file_contents, yaml_contents, section, empty_identifiers) |
|
374
|
|
|
|
|
375
|
|
|
|
|
376
|
|
View Code Duplication |
def fix_empty_reference(file_contents, yaml_contents): |
|
|
|
|
|
|
377
|
|
|
section = 'references' |
|
378
|
|
|
|
|
379
|
|
|
empty_identifiers = [] |
|
380
|
|
|
|
|
381
|
|
|
if yaml_contents[section] is not None: |
|
382
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
|
383
|
|
|
if str(i_value).strip() == "": |
|
384
|
|
|
empty_identifiers.append(i_type) |
|
385
|
|
|
|
|
386
|
|
|
return remove_section_keys(file_contents, yaml_contents, section, empty_identifiers) |
|
387
|
|
|
|
|
388
|
|
|
|
|
389
|
|
|
def fix_prefix_cce(file_contents, yaml_contents): |
|
390
|
|
|
section = 'identifiers' |
|
391
|
|
|
|
|
392
|
|
|
prefixed_identifiers = [] |
|
393
|
|
|
|
|
394
|
|
|
if yaml_contents[section] is not None: |
|
395
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
|
396
|
|
|
if i_type[0:3] == 'cce': |
|
397
|
|
|
has_prefix = i_value[0:3].upper() == 'CCE' |
|
398
|
|
|
remainder_valid = cce.is_cce_format_valid("CCE-" + str(i_value[3:])) |
|
399
|
|
|
remainder_valid |= cce.is_cce_format_valid("CCE-" + str(i_value[4:])) |
|
400
|
|
|
if has_prefix and remainder_valid: |
|
401
|
|
|
prefixed_identifiers.append(i_type) |
|
402
|
|
|
|
|
403
|
|
|
return rewrite_section_value(file_contents, yaml_contents, section, prefixed_identifiers, |
|
404
|
|
|
rewrite_value_remove_prefix) |
|
405
|
|
|
|
|
406
|
|
|
|
|
407
|
|
|
def fix_invalid_cce(file_contents, yaml_contents): |
|
408
|
|
|
section = 'identifiers' |
|
409
|
|
|
|
|
410
|
|
|
invalid_identifiers = [] |
|
411
|
|
|
|
|
412
|
|
|
if yaml_contents[section] is not None: |
|
413
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
|
414
|
|
|
if i_type[0:3] == 'cce': |
|
415
|
|
|
if not cce.is_cce_value_valid("CCE-" + str(i_value)): |
|
416
|
|
|
invalid_identifiers.append(i_type) |
|
417
|
|
|
|
|
418
|
|
|
return remove_section_keys(file_contents, yaml_contents, section, invalid_identifiers) |
|
419
|
|
|
|
|
420
|
|
|
|
|
421
|
|
|
def fix_prodtypes(file_contents, yaml_contents): |
|
422
|
|
|
section = 'prodtype' |
|
423
|
|
|
sorted_prodtypes = yaml_contents[section].split(",") |
|
424
|
|
|
sorted_prodtypes.sort() |
|
425
|
|
|
out = ",".join(sorted_prodtypes) |
|
426
|
|
|
|
|
427
|
|
|
return rewrite_keyless_section(file_contents, yaml_contents, section, out) |
|
428
|
|
|
|
|
429
|
|
|
|
|
430
|
|
|
def has_product_cce(yaml_contents, product): |
|
431
|
|
|
section = 'identifiers' |
|
432
|
|
|
|
|
433
|
|
|
invalid_identifiers = [] |
|
434
|
|
|
|
|
435
|
|
|
if not yaml_contents[section]: |
|
436
|
|
|
return False |
|
437
|
|
|
|
|
438
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
|
439
|
|
|
if i_type[0:3] != 'cce' or "@" not in i_type: |
|
440
|
|
|
continue |
|
441
|
|
|
|
|
442
|
|
|
_, cce_product = i_type.split("@", 1) |
|
443
|
|
|
if product == cce_product: |
|
444
|
|
|
return True |
|
445
|
|
|
|
|
446
|
|
|
return False |
|
447
|
|
|
|
|
448
|
|
|
|
|
449
|
|
|
def add_product_cce(file_contents, yaml_contents, product, cce): |
|
450
|
|
|
section = 'identifiers' |
|
451
|
|
|
|
|
452
|
|
|
if section not in yaml_contents: |
|
453
|
|
|
return file_contents |
|
454
|
|
|
|
|
455
|
|
|
new_contents = add_to_the_section( |
|
456
|
|
|
file_contents, yaml_contents, section, {"cce@{product}".format(product=product): cce}) |
|
457
|
|
|
new_contents = sort_section(new_contents, yaml_contents, section) |
|
458
|
|
|
return new_contents |
|
459
|
|
|
|
|
460
|
|
|
|
|
461
|
|
|
def fix_int_identifier(file_contents, yaml_contents): |
|
462
|
|
|
section = 'identifiers' |
|
463
|
|
|
|
|
464
|
|
|
int_identifiers = [] |
|
465
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
|
466
|
|
|
if type(i_value) != str: |
|
467
|
|
|
int_identifiers.append(i_type) |
|
468
|
|
|
|
|
469
|
|
|
return rewrite_section_value_int_str(file_contents, yaml_contents, section, int_identifiers) |
|
470
|
|
|
|
|
471
|
|
|
|
|
472
|
|
|
def fix_int_reference(file_contents, yaml_contents): |
|
473
|
|
|
section = 'references' |
|
474
|
|
|
|
|
475
|
|
|
int_identifiers = [] |
|
476
|
|
|
for i_type, i_value in yaml_contents[section].items(): |
|
477
|
|
|
if type(i_value) != str: |
|
478
|
|
|
int_identifiers.append(i_type) |
|
479
|
|
|
|
|
480
|
|
|
return rewrite_section_value_int_str(file_contents, yaml_contents, section, int_identifiers) |
|
481
|
|
|
|
|
482
|
|
|
|
|
483
|
|
|
def sort_rule_subkeys(file_contents, yaml_contents): |
|
484
|
|
|
return ssg.rule_yaml.sort_section_keys(None, file_contents, TO_SORT) |
|
485
|
|
|
|
|
486
|
|
|
|
|
487
|
|
|
def _fixed_file_contents(path, file_contents, product_yaml, func): |
|
488
|
|
|
if file_contents[-1] == '': |
|
489
|
|
|
file_contents = file_contents[:-1] |
|
490
|
|
|
|
|
491
|
|
|
subst_dict = product_yaml |
|
492
|
|
|
yaml_contents = yaml.open_and_macro_expand(path, subst_dict) |
|
493
|
|
|
|
|
494
|
|
|
try: |
|
495
|
|
|
new_file_contents = func(file_contents, yaml_contents) |
|
496
|
|
|
except Exception as exc: |
|
497
|
|
|
msg = "Refusing to fix file: {path}: {error}".format(path=path, error=str(exc)) |
|
498
|
|
|
raise RuntimeError(msg) |
|
499
|
|
|
|
|
500
|
|
|
return new_file_contents |
|
501
|
|
|
|
|
502
|
|
|
|
|
503
|
|
|
def fix_file(path, product_yaml, func): |
|
504
|
|
|
file_contents = open(path, 'r').read().split("\n") |
|
505
|
|
|
|
|
506
|
|
|
new_file_contents = _fixed_file_contents(path, file_contents, product_yaml, func) |
|
507
|
|
|
if file_contents == new_file_contents: |
|
508
|
|
|
return False |
|
509
|
|
|
|
|
510
|
|
|
with open(path, 'w') as f: |
|
511
|
|
|
for line in new_file_contents: |
|
512
|
|
|
print(line, file=f) |
|
513
|
|
|
return True |
|
514
|
|
|
|
|
515
|
|
|
|
|
516
|
|
|
def fix_file_prompt(path, product_yaml, func, args): |
|
517
|
|
|
file_contents = open(path, 'r').read().split("\n") |
|
518
|
|
|
|
|
519
|
|
|
new_file_contents = _fixed_file_contents(path, file_contents, product_yaml, func) |
|
520
|
|
|
changes = file_contents != new_file_contents |
|
521
|
|
|
|
|
522
|
|
|
if not changes: |
|
523
|
|
|
return changes |
|
524
|
|
|
|
|
525
|
|
|
need_input = not args.assume_yes and not args.dry_run |
|
526
|
|
|
|
|
527
|
|
|
if need_input: |
|
528
|
|
|
print("====BEGIN BEFORE====") |
|
529
|
|
|
print_file(file_contents) |
|
530
|
|
|
print("====END BEFORE====") |
|
531
|
|
|
|
|
532
|
|
|
if need_input: |
|
533
|
|
|
print("====BEGIN AFTER====") |
|
534
|
|
|
print_file(new_file_contents) |
|
535
|
|
|
print("====END AFTER====") |
|
536
|
|
|
|
|
537
|
|
|
response = 'n' |
|
538
|
|
|
if need_input: |
|
539
|
|
|
response = input_func("Confirm writing output to %s: (y/n): " % path) |
|
540
|
|
|
|
|
541
|
|
|
if args.assume_yes or response.strip().lower() == 'y': |
|
542
|
|
|
changes = True |
|
543
|
|
|
with open(path, 'w') as f: |
|
544
|
|
|
for line in new_file_contents: |
|
545
|
|
|
print(line, file=f) |
|
546
|
|
|
else: |
|
547
|
|
|
changes = False |
|
548
|
|
|
return changes |
|
549
|
|
|
|
|
550
|
|
|
|
|
551
|
|
|
def add_cce(args, product_yaml): |
|
552
|
|
|
directory = os.path.join(args.root, args.subdirectory) |
|
553
|
|
|
cce_pool = cce.CCE_POOLS[args.cce_pool]() |
|
554
|
|
|
return _add_cce(directory, cce_pool, args.rule, product_yaml, args) |
|
555
|
|
|
|
|
556
|
|
|
|
|
557
|
|
|
def _add_cce(directory, cce_pool, rules, product_yaml, args): |
|
558
|
|
|
product = product_yaml["product"] |
|
559
|
|
|
|
|
560
|
|
|
def is_relevant_rule(rule_path, rule, rule_lines): |
|
561
|
|
|
for r in rules: |
|
562
|
|
|
if ( |
|
563
|
|
|
rule_path.endswith("/{r}/rule.yml".format(r=r)) |
|
564
|
|
|
and has_no_cce(rule_path, product_yaml)): |
|
565
|
|
|
return True |
|
566
|
|
|
return False |
|
567
|
|
|
|
|
568
|
|
|
results = find_rules(args, is_relevant_rule) |
|
569
|
|
|
|
|
570
|
|
|
for result in results: |
|
571
|
|
|
rule_path = result[0] |
|
572
|
|
|
|
|
573
|
|
|
cce = cce_pool.random_cce() |
|
574
|
|
|
|
|
575
|
|
|
def fix_callback(file_contents, yaml_contents): |
|
576
|
|
|
return add_product_cce(file_contents, yaml_contents, product_yaml["product"], cce) |
|
|
|
|
|
|
577
|
|
|
|
|
578
|
|
|
try: |
|
579
|
|
|
changes = fix_file(rule_path, product_yaml, fix_callback) |
|
580
|
|
|
except RuntimeError as exc: |
|
581
|
|
|
msg = ( |
|
582
|
|
|
"Error adding CCE into {rule_path}: {exc}" |
|
583
|
|
|
.format(rule_path=rule_path, exc=str(exc))) |
|
584
|
|
|
raise RuntimeError(exc) |
|
585
|
|
|
|
|
586
|
|
|
if changes: |
|
587
|
|
|
cce_pool.remove_cce_from_file(cce) |
|
588
|
|
|
|
|
589
|
|
|
|
|
590
|
|
|
def has_unsorted_prodtype(rule_path, rule, rule_lines): |
|
591
|
|
|
if 'prodtype' in rule: |
|
592
|
|
|
prodtypes = rule['prodtype'].split(',') |
|
593
|
|
|
return prodtypes != sorted(prodtypes) |
|
594
|
|
|
return False |
|
595
|
|
|
|
|
596
|
|
|
|
|
597
|
|
|
@command("empty_identifiers", "check and fix rules with empty identifiers") |
|
598
|
|
|
def fix_empty_identifiers(args, product_yaml): |
|
599
|
|
|
results = find_rules(args, has_empty_identifier) |
|
600
|
|
|
print("Number of rules with empty identifiers: %d" % len(results)) |
|
601
|
|
|
|
|
602
|
|
|
for result in results: |
|
603
|
|
|
rule_path = result[0] |
|
604
|
|
|
|
|
605
|
|
|
product_yaml_path = result[2] |
|
606
|
|
|
|
|
607
|
|
|
if product_yaml_path is not None: |
|
608
|
|
|
product_yaml = yaml.open_raw(product_yaml_path) |
|
609
|
|
|
|
|
610
|
|
|
if args.dry_run: |
|
611
|
|
|
print(rule_path + " has one or more empty identifiers") |
|
612
|
|
|
continue |
|
613
|
|
|
|
|
614
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_empty_identifier, args) |
|
615
|
|
|
|
|
616
|
|
|
exit(int(len(results) > 0)) |
|
617
|
|
|
|
|
618
|
|
|
|
|
619
|
|
|
@command("empty_references", "check and fix rules with empty references") |
|
620
|
|
|
def fix_empty_references(args, product_yaml): |
|
621
|
|
|
results = find_rules(args, has_empty_references) |
|
622
|
|
|
print("Number of rules with empty references: %d" % len(results)) |
|
623
|
|
|
|
|
624
|
|
|
for result in results: |
|
625
|
|
|
rule_path = result[0] |
|
626
|
|
|
product_yaml = result[2] |
|
627
|
|
|
|
|
628
|
|
|
if args.dry_run: |
|
629
|
|
|
print(rule_path + " has one or more empty references") |
|
630
|
|
|
continue |
|
631
|
|
|
|
|
632
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_empty_reference, args) |
|
633
|
|
|
|
|
634
|
|
|
exit(int(len(results) > 0)) |
|
635
|
|
|
|
|
636
|
|
|
|
|
637
|
|
|
@command("prefixed_identifiers", "check and fix rules with prefixed (CCE-) identifiers") |
|
638
|
|
|
def find_prefix_cce(args): |
|
639
|
|
|
results = find_rules(args, has_prefix_cce) |
|
640
|
|
|
print("Number of rules with prefixed CCEs: %d" % len(results)) |
|
641
|
|
|
|
|
642
|
|
|
for result in results: |
|
643
|
|
|
rule_path = result[0] |
|
644
|
|
|
product_yaml = result[2] |
|
645
|
|
|
|
|
646
|
|
|
if args.dry_run: |
|
647
|
|
|
print(rule_path + " has one or more CCE with CCE- prefix") |
|
648
|
|
|
continue |
|
649
|
|
|
|
|
650
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_prefix_cce, args) |
|
651
|
|
|
|
|
652
|
|
|
exit(int(len(results) > 0)) |
|
653
|
|
|
|
|
654
|
|
|
|
|
655
|
|
|
@command("invalid_identifiers", "check and fix rules with invalid identifiers") |
|
656
|
|
|
def find_invalid_cce(args, product_yamls): |
|
657
|
|
|
results = find_rules(args, has_invalid_cce) |
|
658
|
|
|
print("Number of rules with invalid CCEs: %d" % len(results)) |
|
659
|
|
|
|
|
660
|
|
|
for result in results: |
|
661
|
|
|
rule_path = result[0] |
|
662
|
|
|
product_yaml = result[2] |
|
663
|
|
|
|
|
664
|
|
|
if args.dry_run: |
|
665
|
|
|
print(rule_path + " has one or more invalid CCEs") |
|
666
|
|
|
continue |
|
667
|
|
|
|
|
668
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_invalid_cce, args) |
|
669
|
|
|
exit(int(len(results) > 0)) |
|
670
|
|
|
|
|
671
|
|
|
|
|
672
|
|
|
@command("int_identifiers", "check and fix rules with pseudo-integer identifiers") |
|
673
|
|
|
def find_int_identifiers(args, product_yaml): |
|
674
|
|
|
results = find_rules(args, has_int_identifier) |
|
675
|
|
|
print("Number of rules with integer identifiers: %d" % len(results)) |
|
676
|
|
|
|
|
677
|
|
|
for result in results: |
|
678
|
|
|
rule_path = result[0] |
|
679
|
|
|
product_yaml = result[2] |
|
680
|
|
|
|
|
681
|
|
|
if args.dry_run: |
|
682
|
|
|
print(rule_path + " has one or more integer references") |
|
683
|
|
|
continue |
|
684
|
|
|
|
|
685
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_int_identifier, args) |
|
686
|
|
|
|
|
687
|
|
|
exit(int(len(results) > 0)) |
|
688
|
|
|
|
|
689
|
|
|
|
|
690
|
|
|
@command("int_references", "check and fix rules with pseudo-integer references") |
|
691
|
|
|
def find_int_references(args, product_yaml): |
|
692
|
|
|
results = find_rules(args, has_int_reference) |
|
693
|
|
|
print("Number of rules with integer references: %d" % len(results)) |
|
694
|
|
|
|
|
695
|
|
|
for result in results: |
|
696
|
|
|
rule_path = result[0] |
|
697
|
|
|
product_yaml = result[2] |
|
698
|
|
|
|
|
699
|
|
|
if args.dry_run: |
|
700
|
|
|
print(rule_path + " has one or more unsorted references") |
|
701
|
|
|
continue |
|
702
|
|
|
|
|
703
|
|
|
fix_file_prompt(rule_path, product_yaml, fix_int_reference, args) |
|
704
|
|
|
|
|
705
|
|
|
exit(int(len(results) > 0)) |
|
706
|
|
|
|
|
707
|
|
|
|
|
708
|
|
|
@command("duplicate_subkeys", "check for duplicated references and identifiers") |
|
709
|
|
|
def duplicate_subkeys(args, product_yaml): |
|
710
|
|
|
results = find_rules(args, has_duplicated_subkeys) |
|
711
|
|
|
print("Number of rules with duplicated subkeys: %d" % len(results)) |
|
712
|
|
|
|
|
713
|
|
|
for result in results: |
|
714
|
|
|
print(result[0] + " has one or more duplicated subkeys") |
|
715
|
|
|
|
|
716
|
|
|
exit(int(len(results) > 0)) |
|
717
|
|
|
|
|
718
|
|
|
|
|
719
|
|
|
@command("sort_subkeys", "sort references and identifiers") |
|
720
|
|
|
def sort_subkeys(args, product_yaml): |
|
721
|
|
|
results = find_rules(args, has_unordered_sections) |
|
722
|
|
|
print("Number of modified rules: %d" % len(results)) |
|
723
|
|
|
|
|
724
|
|
|
for result in results: |
|
725
|
|
|
rule_path = result[0] |
|
726
|
|
|
product_yaml = result[2] |
|
727
|
|
|
|
|
728
|
|
|
if args.dry_run: |
|
729
|
|
|
print(rule_path + " has one or more unsorted references") |
|
730
|
|
|
continue |
|
731
|
|
|
|
|
732
|
|
|
fix_file_prompt(rule_path, product_yaml, sort_rule_subkeys, args) |
|
733
|
|
|
|
|
734
|
|
|
exit(int(len(results) > 0)) |
|
735
|
|
|
|
|
736
|
|
|
|
|
737
|
|
|
@command("sort_prodtypes", "sorts the products in the prodtype") |
|
738
|
|
|
def sort_prodtypes(args, product_yaml): |
|
739
|
|
|
results = find_rules(args, has_unsorted_prodtype) |
|
740
|
|
|
for result in results: |
|
741
|
|
|
rule_path = result[0] |
|
742
|
|
|
product_yaml = result[2] |
|
743
|
|
|
|
|
744
|
|
|
if args.dry_run: |
|
745
|
|
|
print(rule_path + " prodtype is unsorted") |
|
746
|
|
|
continue |
|
747
|
|
|
|
|
748
|
|
|
fix_file(rule_path, product_yaml, fix_prodtypes) |
|
749
|
|
|
|
|
750
|
|
|
exit(int(len(results) > 0)) |
|
751
|
|
|
|
|
752
|
|
|
|
|
753
|
|
|
@command("test_all", "Perform all checks on all rules") |
|
754
|
|
|
def test_all(args, product_yaml): |
|
755
|
|
|
result = 0 |
|
756
|
|
|
checks = [ |
|
757
|
|
|
(has_empty_identifier, "empty identifiers"), |
|
758
|
|
|
(has_invalid_cce, "invalid CCEs"), |
|
759
|
|
|
(has_int_identifier, "integer references"), |
|
760
|
|
|
(has_empty_references, "empty references"), |
|
761
|
|
|
(has_int_reference, "unsorted references"), |
|
762
|
|
|
(has_duplicated_subkeys, "duplicated subkeys"), |
|
763
|
|
|
(has_unordered_sections, "unsorted references"), |
|
764
|
|
|
(has_unsorted_prodtype, "unsorted prodtype") |
|
765
|
|
|
] |
|
766
|
|
|
for item in rule_data_generator(args): |
|
767
|
|
|
rule_path, rule, rule_lines, _, _ = item |
|
768
|
|
|
for func, msg in checks: |
|
769
|
|
|
if func(rule_path, rule, rule_lines): |
|
770
|
|
|
print("Rule '%s' has %s" % (rule_path, msg)) |
|
771
|
|
|
result = 1 |
|
772
|
|
|
exit(result) |
|
773
|
|
|
|
|
774
|
|
|
|
|
775
|
|
|
def create_parser_from_functions(subparsers): |
|
776
|
|
|
for name, function in _COMMANDS.items(): |
|
777
|
|
|
subparser = subparsers.add_parser(name, description=function.description) |
|
778
|
|
|
subparser.set_defaults(func=function) |
|
779
|
|
|
|
|
780
|
|
|
|
|
781
|
|
|
def create_other_parsers(subparsers): |
|
782
|
|
|
subparser = subparsers.add_parser("add-cce", description="Add CCE to rule files") |
|
783
|
|
|
subparser.add_argument("rule", nargs="+") |
|
784
|
|
|
subparser.add_argument("--subdirectory", default="linux_os") |
|
785
|
|
|
subparser.add_argument( |
|
786
|
|
|
"--cce-pool", "-p", default="redhat", choices=list(cce.CCE_POOLS.keys()), |
|
787
|
|
|
) |
|
788
|
|
|
subparser.set_defaults(func=add_cce) |
|
789
|
|
|
|
|
790
|
|
|
|
|
791
|
|
|
def parse_args(): |
|
792
|
|
|
parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, |
|
793
|
|
|
description="Utility for fixing mistakes in rule files") |
|
794
|
|
|
parser.add_argument( |
|
795
|
|
|
"-y", "--assume-yes", default=False, action="store_true", |
|
796
|
|
|
help="Assume yes and overwrite all files (no prompt)") |
|
797
|
|
|
parser.add_argument( |
|
798
|
|
|
"-d", "--dry-run", default=False, action="store_true", |
|
799
|
|
|
help="Assume no and don't overwrite any files") |
|
800
|
|
|
parser.add_argument( |
|
801
|
|
|
"-j", "--json", type=str, action="store", |
|
802
|
|
|
default="build/rule_dirs.json", help="File to read json " |
|
803
|
|
|
"output of rule_dir_json.py from (defaults to " |
|
804
|
|
|
"build/rule_dirs.json") |
|
805
|
|
|
parser.add_argument( |
|
806
|
|
|
"-r", "--root", default=SSG_ROOT, |
|
807
|
|
|
help="Path to root of the project directory") |
|
808
|
|
|
parser.add_argument("--product", "-p", help="Path to the main product.yml") |
|
809
|
|
|
subparsers = parser.add_subparsers(title="command", help="What to perform.") |
|
810
|
|
|
subparsers.required = True |
|
811
|
|
|
create_parser_from_functions(subparsers) |
|
812
|
|
|
create_other_parsers(subparsers) |
|
813
|
|
|
return parser.parse_args() |
|
814
|
|
|
|
|
815
|
|
|
|
|
816
|
|
|
def __main__(): |
|
817
|
|
|
args = parse_args() |
|
818
|
|
|
project_root = args.root |
|
819
|
|
|
if not project_root: |
|
820
|
|
|
project_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.pardir) |
|
821
|
|
|
|
|
822
|
|
|
subst_dict = dict() |
|
823
|
|
|
if args.product: |
|
824
|
|
|
subst_dict = dict() |
|
825
|
|
|
product = products.load_product_yaml(args.product) |
|
826
|
|
|
product.read_properties_from_directory(os.path.join(project_root, "product_properties")) |
|
827
|
|
|
subst_dict.update(product) |
|
828
|
|
|
|
|
829
|
|
|
args.func(args, subst_dict) |
|
830
|
|
|
|
|
831
|
|
|
|
|
832
|
|
|
if __name__ == "__main__": |
|
833
|
|
|
__main__() |
|
834
|
|
|
|