1
|
|
|
from __future__ import absolute_import |
2
|
|
|
from __future__ import print_function |
3
|
|
|
|
4
|
|
|
import os |
5
|
|
|
import os.path |
6
|
|
|
import sys |
7
|
|
|
import re |
8
|
|
|
from copy import deepcopy |
9
|
|
|
import collections |
10
|
|
|
|
11
|
|
|
from .build_yaml import Rule, DocumentationNotComplete |
12
|
|
|
from .constants import oval_namespace as oval_ns |
13
|
|
|
from .constants import oval_footer |
14
|
|
|
from .constants import oval_header |
15
|
|
|
from .constants import MULTI_PLATFORM_LIST |
16
|
|
|
from .jinja import process_file_with_macros |
17
|
|
|
from .rule_yaml import parse_prodtype |
18
|
|
|
from .rules import get_rule_dir_id, get_rule_dir_ovals, find_rule_dirs_in_paths |
19
|
|
|
from . import utils |
20
|
|
|
from .xml import ElementTree |
21
|
|
|
|
22
|
|
|
|
23
|
|
|
def _check_is_applicable_for_product(oval_check_def, product): |
24
|
|
|
"""Based on the <platform> specifier of the OVAL check determine if this |
25
|
|
|
OVAL check is applicable for this product. Return 'True' if so, 'False' |
26
|
|
|
otherwise""" |
27
|
|
|
|
28
|
|
|
product, product_version = utils.parse_name(product) |
29
|
|
|
|
30
|
|
|
# Define general platforms |
31
|
|
|
multi_platforms = ['<platform>multi_platform_all', |
32
|
|
|
'<platform>multi_platform_' + product] |
33
|
|
|
|
34
|
|
|
# First test if OVAL check isn't for 'multi_platform_all' or |
35
|
|
|
# 'multi_platform_' + product |
36
|
|
|
for multi_prod in multi_platforms: |
37
|
|
|
if multi_prod in oval_check_def and product in MULTI_PLATFORM_LIST: |
38
|
|
|
return True |
39
|
|
|
|
40
|
|
|
# Current SSG checks aren't unified which element of '<platform>' |
41
|
|
|
# and '<product>' to use as OVAL AffectedType metadata element, |
42
|
|
|
# e.g. Chromium content uses both of them across the various checks |
43
|
|
|
# Thus for now check both of them when checking concrete platform / product |
44
|
|
|
affected_type_elements = ['<platform>', '<product>'] |
45
|
|
|
|
46
|
|
|
for afftype in affected_type_elements: |
47
|
|
|
# Get official name for product (prefixed with content of afftype) |
48
|
|
|
product_name = afftype + utils.map_name(product) |
49
|
|
|
# Append the product version to the official name |
50
|
|
|
if product_version is not None: |
51
|
|
|
# Some product versions have a dot in between the numbers |
52
|
|
|
# While the prodtype doesn't have the dot, the full product name does |
53
|
|
|
if product == "ubuntu" or product == "macos": |
54
|
|
|
product_version = product_version[:2] + "." + product_version[2:] |
55
|
|
|
product_name += ' ' + product_version |
56
|
|
|
|
57
|
|
|
# Test if this OVAL check is for the concrete product version |
58
|
|
|
if product_name in oval_check_def: |
59
|
|
|
return True |
60
|
|
|
|
61
|
|
|
# OVAL check isn't neither a multi platform one, nor isn't applicable |
62
|
|
|
# for this product => return False to indicate that |
63
|
|
|
|
64
|
|
|
return False |
65
|
|
|
|
66
|
|
|
|
67
|
|
|
def finalize_affected_platforms(xml_tree, env_yaml): |
68
|
|
|
"""Depending on your use-case of OVAL you may not need the <affected> |
69
|
|
|
element. Such use-cases including using OVAL as a check engine for XCCDF |
70
|
|
|
benchmarks. Since the XCCDF Benchmarks use cpe:platform with CPE IDs, |
71
|
|
|
the affected element in OVAL definitions is redundant and just bloats the |
72
|
|
|
files. This function removes all *irrelevant* affected platform elements |
73
|
|
|
from given OVAL tree. It then adds one platform of the product we are |
74
|
|
|
building. |
75
|
|
|
""" |
76
|
|
|
|
77
|
|
|
for affected in xml_tree.findall(".//{%s}affected" % (oval_ns)): |
78
|
|
|
for platform in affected.findall("./{%s}platform" % (oval_ns)): |
79
|
|
|
affected.remove(platform) |
80
|
|
|
for product in affected.findall("./{%s}product" % (oval_ns)): |
81
|
|
|
affected.remove(product) |
82
|
|
|
|
83
|
|
|
final = ElementTree.SubElement( |
84
|
|
|
affected, "{%s}%s" % (oval_ns, utils.required_key(env_yaml, "type"))) |
85
|
|
|
final.text = utils.required_key(env_yaml, "full_name") |
86
|
|
|
|
87
|
|
|
return xml_tree |
88
|
|
|
|
89
|
|
|
|
90
|
|
|
def oval_entities_are_identical(firstelem, secondelem): |
91
|
|
|
"""Check if OVAL entities represented by XML elements are identical |
92
|
|
|
Return: True if identical, False otherwise |
93
|
|
|
Based on: http://stackoverflow.com/a/24349916""" |
94
|
|
|
|
95
|
|
|
# Per https://github.com/ComplianceAsCode/content/pull/1343#issuecomment-234541909 |
96
|
|
|
# and https://github.com/ComplianceAsCode/content/pull/1343#issuecomment-234545296 |
97
|
|
|
# ignore the differences in 'comment', 'version', 'state_operator', and |
98
|
|
|
# 'deprecated' attributes. Also ignore different nsmap, since all these |
99
|
|
|
# don't affect the semantics of the OVAL entities |
100
|
|
|
|
101
|
|
|
# Operate on copies of the elements (since we will modify |
102
|
|
|
# some attributes). Deepcopy will also reset the namespace map |
103
|
|
|
# on copied elements for us |
104
|
|
|
firstcopy = deepcopy(firstelem) |
105
|
|
|
secondcopy = deepcopy(secondelem) |
106
|
|
|
|
107
|
|
|
# Ignore 'comment', 'version', 'state_operator', and 'deprecated' |
108
|
|
|
# attributes since they don't change the semantics of an element |
109
|
|
|
for copy in [firstcopy, secondcopy]: |
110
|
|
|
for key in copy.keys(): |
111
|
|
|
if key in ["comment", "version", "state_operator", |
112
|
|
|
"deprecated"]: |
113
|
|
|
del copy.attrib[key] |
114
|
|
|
|
115
|
|
|
# Compare the equality of the copies |
116
|
|
|
if firstcopy.tag != secondcopy.tag: |
117
|
|
|
return False |
118
|
|
|
if firstcopy.text != secondcopy.text: |
119
|
|
|
return False |
120
|
|
|
if firstcopy.tail != secondcopy.tail: |
121
|
|
|
return False |
122
|
|
|
if firstcopy.attrib != secondcopy.attrib: |
123
|
|
|
return False |
124
|
|
|
if len(firstcopy) != len(secondcopy): |
125
|
|
|
return False |
126
|
|
|
|
127
|
|
|
return all(oval_entities_are_identical( |
128
|
|
|
fchild, schild) for fchild, schild in zip(firstcopy, secondcopy)) |
129
|
|
|
|
130
|
|
|
|
131
|
|
|
def oval_entity_is_extvar(elem): |
132
|
|
|
"""Check if OVAL entity represented by XML element is OVAL |
133
|
|
|
<external_variable> element |
134
|
|
|
Return: True if <external_variable>, False otherwise""" |
135
|
|
|
|
136
|
|
|
return elem.tag == '{%s}external_variable' % oval_ns |
137
|
|
|
|
138
|
|
|
|
139
|
|
|
element_child_cache = collections.defaultdict(dict) |
140
|
|
|
|
141
|
|
|
|
142
|
|
|
def append(element, newchild): |
143
|
|
|
"""Append new child ONLY if it's not a duplicate""" |
144
|
|
|
|
145
|
|
|
global element_child_cache |
146
|
|
|
|
147
|
|
|
newid = newchild.get("id") |
148
|
|
|
existing = element_child_cache[element].get(newid, None) |
149
|
|
|
|
150
|
|
|
if existing is not None: |
151
|
|
|
# ID is identical and OVAL entities are identical |
152
|
|
|
if oval_entities_are_identical(existing, newchild): |
153
|
|
|
# Moreover the entity is OVAL <external_variable> |
154
|
|
|
if oval_entity_is_extvar(newchild): |
155
|
|
|
# If OVAL entity is identical to some already included |
156
|
|
|
# in the benchmark and represents an OVAL <external_variable> |
157
|
|
|
# it's safe to ignore this ID (since external variables are |
158
|
|
|
# in multiple checks just to notify 'testoval.py' helper to |
159
|
|
|
# substitute the ID with <local_variable> entity when testing |
160
|
|
|
# the OVAL for the rule) |
161
|
|
|
pass |
162
|
|
|
# Some other OVAL entity |
163
|
|
|
else: |
164
|
|
|
# If OVAL entity is identical, but not external_variable, the |
165
|
|
|
# implementation should be rewritten each entity to be present |
166
|
|
|
# just once |
167
|
|
|
sys.stderr.write("ERROR: OVAL ID '%s' is used multiple times " |
168
|
|
|
"and should represent the same elements.\n" |
169
|
|
|
% (newid)) |
170
|
|
|
sys.stderr.write("Rewrite the OVAL checks. Place the identical " |
171
|
|
|
"IDs into their own definition and extend " |
172
|
|
|
"this definition by it.\n") |
173
|
|
|
sys.exit(1) |
174
|
|
|
# ID is identical, but OVAL entities are semantically difference => |
175
|
|
|
# report and error and exit with failure |
176
|
|
|
# Fixes: https://github.com/ComplianceAsCode/content/issues/1275 |
177
|
|
|
else: |
178
|
|
|
if not oval_entity_is_extvar(existing) and \ |
179
|
|
|
not oval_entity_is_extvar(newchild): |
180
|
|
|
# This is an error scenario - since by skipping second |
181
|
|
|
# implementation and using the first one for both references, |
182
|
|
|
# we might evaluate wrong requirement for the second entity |
183
|
|
|
# => report an error and exit with failure in that case |
184
|
|
|
# See |
185
|
|
|
# https://github.com/ComplianceAsCode/content/issues/1275 |
186
|
|
|
# for a reproducer and what could happen in this case |
187
|
|
|
sys.stderr.write("ERROR: it's not possible to use the " + |
188
|
|
|
"same ID: %s " % newid + "for two " + |
189
|
|
|
"semantically different OVAL entities:\n") |
190
|
|
|
sys.stderr.write("First entity %s\n" % ElementTree.tostring(existing)) |
191
|
|
|
sys.stderr.write("Second entity %s\n" % ElementTree.tostring(newchild)) |
192
|
|
|
sys.stderr.write("Use different ID for the second entity!!!\n") |
193
|
|
|
sys.exit(1) |
194
|
|
|
else: |
195
|
|
|
element.append(newchild) |
196
|
|
|
element_child_cache[element][newid] = newchild |
197
|
|
|
|
198
|
|
|
|
199
|
|
|
def check_oval_version(oval_version): |
200
|
|
|
"""Not necessary, but should help with typos""" |
201
|
|
|
|
202
|
|
|
supported_versions = ["5.10", "5.11"] |
203
|
|
|
if oval_version not in supported_versions: |
204
|
|
|
supported_versions_str = ", ".join(supported_versions) |
205
|
|
|
sys.stderr.write( |
206
|
|
|
"Suspicious oval version \"%s\", one of {%s} is " |
207
|
|
|
"expected.\n" % (oval_version, supported_versions_str)) |
208
|
|
|
sys.exit(1) |
209
|
|
|
|
210
|
|
|
|
211
|
|
|
def _check_is_loaded(loaded_dict, filename, version): |
212
|
|
|
if filename in loaded_dict: |
213
|
|
|
if loaded_dict[filename] >= version: |
214
|
|
|
return True |
215
|
|
|
|
216
|
|
|
# Should rather fail, than override something unwanted |
217
|
|
|
sys.stderr.write( |
218
|
|
|
"You cannot override generic OVAL file in version '%s' " |
219
|
|
|
"by more specific one in older version '%s'" % |
220
|
|
|
(version, loaded_dict[filename]) |
221
|
|
|
) |
222
|
|
|
sys.exit(1) |
223
|
|
|
|
224
|
|
|
return False |
225
|
|
|
|
226
|
|
|
|
227
|
|
|
def _create_oval_tree_from_string(xml_content): |
228
|
|
|
try: |
229
|
|
|
argument = oval_header + xml_content + oval_footer |
230
|
|
|
oval_file_tree = ElementTree.fromstring(argument) |
231
|
|
|
except ElementTree.ParseError as error: |
232
|
|
|
line, column = error.position |
233
|
|
|
lines = argument.splitlines() |
234
|
|
|
before = '\n'.join(lines[:line]) |
235
|
|
|
column_pointer = ' ' * (column - 1) + '^' |
236
|
|
|
sys.stderr.write( |
237
|
|
|
"%s\n%s\nError when parsing OVAL file.\n" % |
238
|
|
|
(before, column_pointer)) |
239
|
|
|
sys.exit(1) |
240
|
|
|
return oval_file_tree |
241
|
|
|
|
242
|
|
|
|
243
|
|
|
def _check_oval_version_from_oval(oval_file_tree, oval_version): |
244
|
|
|
for defgroup in oval_file_tree.findall("./{%s}def-group" % oval_ns): |
245
|
|
|
file_oval_version = defgroup.get("oval_version") |
246
|
|
|
|
247
|
|
|
if file_oval_version is None: |
|
|
|
|
248
|
|
|
# oval_version does not exist in <def-group/> |
249
|
|
|
# which means the OVAL is supported for any version. |
250
|
|
|
# By default, that version is 5.10 |
251
|
|
|
file_oval_version = "5.10" |
252
|
|
|
|
253
|
|
|
if tuple(oval_version.split(".")) >= tuple(file_oval_version.split(".")): |
254
|
|
|
return True |
255
|
|
|
|
256
|
|
|
|
257
|
|
|
def _check_rule_id(oval_file_tree, rule_id): |
258
|
|
|
for definition in oval_file_tree.findall( |
259
|
|
|
"./{%s}def-group/{%s}definition" % (oval_ns, oval_ns)): |
260
|
|
|
definition_id = definition.get("id") |
261
|
|
|
return definition_id == rule_id |
262
|
|
|
return False |
263
|
|
|
|
264
|
|
|
|
265
|
|
|
def checks(env_yaml, yaml_path, oval_version, oval_dirs): |
266
|
|
|
""" |
267
|
|
|
Concatenate all XML files in the oval directory, to create the document |
268
|
|
|
body. Then concatenates this with all XML files in the guide directories, |
269
|
|
|
preferring {{{ product }}}.xml to shared.xml. |
270
|
|
|
|
271
|
|
|
oval_dirs: list of directory with oval files (later has higher priority) |
272
|
|
|
|
273
|
|
|
Return: The document body |
274
|
|
|
""" |
275
|
|
|
|
276
|
|
|
body = [] |
277
|
|
|
product = utils.required_key(env_yaml, "product") |
278
|
|
|
included_checks_count = 0 |
279
|
|
|
reversed_dirs = oval_dirs[::-1] # earlier directory has higher priority |
280
|
|
|
already_loaded = dict() # filename -> oval_version |
281
|
|
|
local_env_yaml = dict() |
282
|
|
|
local_env_yaml.update(env_yaml) |
283
|
|
|
|
284
|
|
|
product_dir = os.path.dirname(yaml_path) |
285
|
|
|
relative_guide_dir = utils.required_key(env_yaml, "benchmark_root") |
286
|
|
|
guide_dir = os.path.abspath(os.path.join(product_dir, relative_guide_dir)) |
287
|
|
|
additional_content_directories = env_yaml.get("additional_content_directories", []) |
288
|
|
|
add_content_dirs = [os.path.abspath(os.path.join(product_dir, rd)) for rd in additional_content_directories] |
289
|
|
|
|
290
|
|
|
for _dir_path in find_rule_dirs_in_paths([guide_dir] + add_content_dirs): |
291
|
|
|
rule_id = get_rule_dir_id(_dir_path) |
292
|
|
|
|
293
|
|
|
rule_path = os.path.join(_dir_path, "rule.yml") |
294
|
|
|
try: |
295
|
|
|
rule = Rule.from_yaml(rule_path, env_yaml) |
296
|
|
|
except DocumentationNotComplete: |
297
|
|
|
# Happens on non-debug build when a rule is "documentation-incomplete" |
298
|
|
|
continue |
299
|
|
|
prodtypes = parse_prodtype(rule.prodtype) |
300
|
|
|
|
301
|
|
|
local_env_yaml['rule_id'] = rule.id_ |
302
|
|
|
local_env_yaml['rule_title'] = rule.title |
303
|
|
|
local_env_yaml['products'] = prodtypes # default is all |
304
|
|
|
|
305
|
|
|
for _path in get_rule_dir_ovals(_dir_path, product): |
306
|
|
|
# To be compatible with the later checks, use the rule_id |
307
|
|
|
# (i.e., the value of _dir) to recreate the expected filename if |
308
|
|
|
# this OVAL was in a rule directory. |
309
|
|
|
filename = "%s.xml" % rule_id |
310
|
|
|
|
311
|
|
|
xml_content = process_file_with_macros(_path, local_env_yaml) |
312
|
|
|
|
313
|
|
|
if not _check_is_applicable_for_product(xml_content, product): |
314
|
|
|
continue |
315
|
|
|
if _check_is_loaded(already_loaded, filename, oval_version): |
316
|
|
|
continue |
317
|
|
|
oval_file_tree = _create_oval_tree_from_string(xml_content) |
318
|
|
|
if not _check_rule_id(oval_file_tree, rule_id,): |
319
|
|
|
msg = "OVAL definition in '%s' doesn't match rule ID '%s'." % ( |
320
|
|
|
_path, rule_id) |
321
|
|
|
print(msg, file=sys.stderr) |
322
|
|
|
if not _check_oval_version_from_oval(oval_file_tree, oval_version): |
323
|
|
|
continue |
324
|
|
|
|
325
|
|
|
body.append(xml_content) |
326
|
|
|
included_checks_count += 1 |
327
|
|
|
already_loaded[filename] = oval_version |
328
|
|
|
|
329
|
|
|
for oval_dir in reversed_dirs: |
330
|
|
|
if os.path.isdir(oval_dir): |
331
|
|
|
# sort the files to make output deterministic |
332
|
|
|
for filename in sorted(os.listdir(oval_dir)): |
333
|
|
|
if filename.endswith(".xml"): |
334
|
|
|
xml_content = process_file_with_macros( |
335
|
|
|
os.path.join(oval_dir, filename), env_yaml |
336
|
|
|
) |
337
|
|
|
|
338
|
|
|
if not _check_is_applicable_for_product(xml_content, product): |
339
|
|
|
continue |
340
|
|
|
if _check_is_loaded(already_loaded, filename, oval_version): |
341
|
|
|
continue |
342
|
|
|
oval_file_tree = _create_oval_tree_from_string(xml_content) |
343
|
|
|
if not _check_oval_version_from_oval(oval_file_tree, oval_version): |
344
|
|
|
continue |
345
|
|
|
body.append(xml_content) |
346
|
|
|
included_checks_count += 1 |
347
|
|
|
already_loaded[filename] = oval_version |
348
|
|
|
|
349
|
|
|
return "".join(body) |
350
|
|
|
|