|
1
|
|
|
from __future__ import absolute_import |
|
2
|
|
|
from __future__ import print_function |
|
3
|
|
|
|
|
4
|
|
|
import os |
|
5
|
|
|
import sys |
|
6
|
|
|
import imp |
|
7
|
|
|
import glob |
|
8
|
|
|
|
|
9
|
|
|
import ssg.build_yaml |
|
10
|
|
|
import ssg.utils |
|
11
|
|
|
import ssg.yaml |
|
12
|
|
|
|
|
13
|
|
|
try: |
|
14
|
|
|
from urllib.parse import quote |
|
15
|
|
|
except ImportError: |
|
16
|
|
|
from urllib import quote |
|
17
|
|
|
|
|
18
|
|
|
languages = ["anaconda", "ansible", "bash", "oval", "puppet", "ignition", "kubernetes"] |
|
19
|
|
|
preprocessing_file_name = "template.py" |
|
20
|
|
|
lang_to_ext_map = { |
|
21
|
|
|
"anaconda": ".anaconda", |
|
22
|
|
|
"ansible": ".yml", |
|
23
|
|
|
"bash": ".sh", |
|
24
|
|
|
"oval": ".xml", |
|
25
|
|
|
"puppet": ".pp", |
|
26
|
|
|
"ignition": ".yml", |
|
27
|
|
|
"kubernetes": ".yml" |
|
28
|
|
|
} |
|
29
|
|
|
|
|
30
|
|
|
|
|
31
|
|
|
templates = dict() |
|
32
|
|
|
|
|
33
|
|
|
|
|
34
|
|
|
class Template(): |
|
35
|
|
|
def __init__(self, template_root_directory, name): |
|
36
|
|
|
self.template_root_directory = template_root_directory |
|
37
|
|
|
self.name = name |
|
38
|
|
|
self.template_path = os.path.join(self.template_root_directory, self.name) |
|
39
|
|
|
self.template_yaml_path = os.path.join(self.template_path, "template.yml") |
|
40
|
|
|
self.preprocessing_file_path = os.path.join(self.template_path, preprocessing_file_name) |
|
41
|
|
|
|
|
42
|
|
|
def load(self): |
|
43
|
|
|
if not os.path.exists(self.preprocessing_file_path): |
|
44
|
|
|
self.preprocessing_file_path = None |
|
45
|
|
|
self.langs = [] |
|
46
|
|
|
template_yaml = ssg.yaml.open_raw(self.template_yaml_path) |
|
47
|
|
|
for lang in template_yaml["supported_languages"]: |
|
48
|
|
|
if lang not in languages: |
|
49
|
|
|
raise ValueError("The template {0} declares to support the {1} language," |
|
50
|
|
|
"but this language is not supported by the content.".format(self.name, lang)) |
|
51
|
|
|
langfilename = lang + ".template" |
|
52
|
|
|
if not os.path.exists(os.path.join(self.template_path, langfilename)): |
|
53
|
|
|
raise ValueError("The template {0} declares to support the {1} language," |
|
54
|
|
|
"but the implementation file is missing.".format(self.name, lang)) |
|
55
|
|
|
self.langs.append(lang) |
|
56
|
|
|
|
|
57
|
|
|
def preprocess(self, parameters, lang): |
|
58
|
|
|
# if no template.py file exists, skip this preprocessing part |
|
59
|
|
|
if self.preprocessing_file_path is not None: |
|
60
|
|
|
module_name = "tmpmod" # dummy module name, we don't need it later |
|
61
|
|
|
preprocess_mod = imp.load_source(module_name, self.preprocessing_file_path) |
|
62
|
|
|
parameters = preprocess_mod.preprocess(parameters.copy(), lang) |
|
63
|
|
|
# TODO: Remove this right after the variables in templates are renamed |
|
64
|
|
|
# to lowercase |
|
65
|
|
|
uppercases = dict() |
|
66
|
|
|
for k, v in parameters.items(): |
|
67
|
|
|
uppercases[k.upper()] = v |
|
68
|
|
|
return uppercases |
|
69
|
|
|
|
|
70
|
|
|
def looks_like_template(self): |
|
71
|
|
|
if not os.path.isdir(self.template_root_directory): |
|
72
|
|
|
return False |
|
73
|
|
|
if os.path.islink(self.template_root_directory): |
|
74
|
|
|
return False |
|
75
|
|
|
template_sources = glob.glob(os.path.join(self.template_path, "*.template")) |
|
76
|
|
|
if not os.path.isfile(self.template_yaml_path) and not template_sources: |
|
77
|
|
|
return False |
|
78
|
|
|
return True |
|
79
|
|
|
|
|
80
|
|
|
|
|
81
|
|
|
class Builder(object): |
|
82
|
|
|
""" |
|
83
|
|
|
Class for building all templated content for a given product. |
|
84
|
|
|
|
|
85
|
|
|
To generate content from templates, pass the env_yaml, path to the |
|
86
|
|
|
directory with resolved rule YAMLs, path to the directory that contains |
|
87
|
|
|
templates, path to the output directory for checks and a path to the |
|
88
|
|
|
output directory for remediations into the constructor. Then, call the |
|
89
|
|
|
method build() to perform a build. |
|
90
|
|
|
""" |
|
91
|
|
|
def __init__( |
|
92
|
|
|
self, env_yaml, resolved_rules_dir, templates_dir, |
|
93
|
|
|
remediations_dir, checks_dir): |
|
94
|
|
|
self.env_yaml = env_yaml |
|
95
|
|
|
self.resolved_rules_dir = resolved_rules_dir |
|
96
|
|
|
self.templates_dir = templates_dir |
|
97
|
|
|
self.remediations_dir = remediations_dir |
|
98
|
|
|
self.checks_dir = checks_dir |
|
99
|
|
|
self.output_dirs = dict() |
|
100
|
|
|
for lang in languages: |
|
101
|
|
|
if lang == "oval": |
|
102
|
|
|
# OVAL checks need to be put to a different directory because |
|
103
|
|
|
# they are processed differently than remediations later in the |
|
104
|
|
|
# build process |
|
105
|
|
|
output_dir = self.checks_dir |
|
106
|
|
|
else: |
|
107
|
|
|
output_dir = self.remediations_dir |
|
108
|
|
|
dir_ = os.path.join(output_dir, lang) |
|
109
|
|
|
self.output_dirs[lang] = dir_ |
|
110
|
|
|
# scan directory structure and dynamically create list of templates |
|
111
|
|
|
for item in os.listdir(self.templates_dir): |
|
112
|
|
|
itempath = os.path.join(self.templates_dir, item) |
|
113
|
|
|
maybe_template = Template(templates_dir, item) |
|
114
|
|
|
if maybe_template.looks_like_template(): |
|
115
|
|
|
maybe_template.load() |
|
116
|
|
|
templates[item] = maybe_template |
|
117
|
|
|
|
|
118
|
|
|
|
|
119
|
|
|
def build_lang( |
|
120
|
|
|
self, rule_id, template_name, template_vars, lang, local_env_yaml): |
|
121
|
|
|
""" |
|
122
|
|
|
Builds templated content for a given rule for a given language. |
|
123
|
|
|
Writes the output to the correct build directories. |
|
124
|
|
|
""" |
|
125
|
|
|
if lang not in templates[template_name].langs: |
|
126
|
|
|
return |
|
127
|
|
|
template_file_name = lang + ".template" |
|
128
|
|
|
template_file_path = os.path.join(self.templates_dir, template_name, template_file_name) |
|
129
|
|
|
ext = lang_to_ext_map[lang] |
|
130
|
|
|
output_file_name = rule_id + ext |
|
131
|
|
|
output_filepath = os.path.join( |
|
132
|
|
|
self.output_dirs[lang], output_file_name) |
|
133
|
|
|
template_parameters = templates[template_name].preprocess(template_vars, lang) |
|
134
|
|
|
jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters) |
|
135
|
|
|
filled_template = ssg.jinja.process_file_with_macros( |
|
136
|
|
|
template_file_path, jinja_dict) |
|
137
|
|
|
with open(output_filepath, "w") as f: |
|
138
|
|
|
f.write(filled_template) |
|
139
|
|
|
|
|
140
|
|
|
def get_langs_to_generate(self, rule): |
|
141
|
|
|
""" |
|
142
|
|
|
For a given rule returns list of languages that should be generated |
|
143
|
|
|
from templates. This is controlled by "template_backends" in rule.yml. |
|
144
|
|
|
""" |
|
145
|
|
|
if "backends" in rule.template: |
|
146
|
|
|
backends = rule.template["backends"] |
|
147
|
|
|
for lang in backends: |
|
148
|
|
|
if lang not in languages: |
|
149
|
|
|
raise RuntimeError( |
|
150
|
|
|
"Rule {0} wants to generate unknown language '{1}" |
|
151
|
|
|
"from a template.".format(rule.id_, lang) |
|
152
|
|
|
) |
|
153
|
|
|
langs_to_generate = [] |
|
154
|
|
|
for lang in languages: |
|
155
|
|
|
backend = backends.get(lang, "on") |
|
156
|
|
|
if backend == "on": |
|
157
|
|
|
langs_to_generate.append(lang) |
|
158
|
|
|
return langs_to_generate |
|
159
|
|
|
else: |
|
160
|
|
|
return languages |
|
161
|
|
|
|
|
162
|
|
|
def build_rule(self, rule_id, rule_title, template, langs_to_generate): |
|
163
|
|
|
""" |
|
164
|
|
|
Builds templated content for a given rule for selected languages, |
|
165
|
|
|
writing the output to the correct build directories. |
|
166
|
|
|
""" |
|
167
|
|
|
try: |
|
168
|
|
|
template_name = template["name"] |
|
169
|
|
|
except KeyError: |
|
170
|
|
|
raise ValueError( |
|
171
|
|
|
"Rule {0} is missing template name under template key".format( |
|
172
|
|
|
rule_id)) |
|
173
|
|
|
if template_name not in templates.keys(): |
|
174
|
|
|
raise ValueError( |
|
175
|
|
|
"Rule {0} uses template {1} which does not exist.".format( |
|
176
|
|
|
rule_id, template_name)) |
|
177
|
|
|
try: |
|
178
|
|
|
template_vars = template["vars"] |
|
179
|
|
|
except KeyError: |
|
180
|
|
|
raise ValueError( |
|
181
|
|
|
"Rule {0} does not contain mandatory 'vars:' key under " |
|
182
|
|
|
"'template:' key.".format(rule_id)) |
|
183
|
|
|
# Add the rule ID which will be reused in OVAL templates as OVAL |
|
184
|
|
|
# definition ID so that the build system matches the generated |
|
185
|
|
|
# check with the rule. |
|
186
|
|
|
template_vars["_rule_id"] = rule_id |
|
187
|
|
|
# checks and remediations are processed with a custom YAML dict |
|
188
|
|
|
local_env_yaml = self.env_yaml.copy() |
|
189
|
|
|
local_env_yaml["rule_id"] = rule_id |
|
190
|
|
|
local_env_yaml["rule_title"] = rule_title |
|
191
|
|
|
local_env_yaml["products"] = self.env_yaml["product"] |
|
192
|
|
|
for lang in langs_to_generate: |
|
193
|
|
|
self.build_lang( |
|
194
|
|
|
rule_id, template_name, template_vars, lang, local_env_yaml) |
|
195
|
|
|
|
|
196
|
|
|
def build_extra_ovals(self): |
|
197
|
|
|
declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml") |
|
198
|
|
|
declaration = ssg.yaml.open_raw(declaration_path) |
|
199
|
|
|
for oval_def_id, template in declaration.items(): |
|
200
|
|
|
langs_to_generate = ["oval"] |
|
201
|
|
|
# Since OVAL definition ID in shorthand format is always the same |
|
202
|
|
|
# as rule ID, we can use it instead of the rule ID even if no rule |
|
203
|
|
|
# with that ID exists |
|
204
|
|
|
self.build_rule( |
|
205
|
|
|
oval_def_id, oval_def_id, template, langs_to_generate) |
|
206
|
|
|
|
|
207
|
|
|
def build_all_rules(self): |
|
208
|
|
|
for rule_file in os.listdir(self.resolved_rules_dir): |
|
209
|
|
|
rule_path = os.path.join(self.resolved_rules_dir, rule_file) |
|
210
|
|
|
try: |
|
211
|
|
|
rule = ssg.build_yaml.Rule.from_yaml(rule_path, self.env_yaml) |
|
212
|
|
|
except ssg.build_yaml.DocumentationNotComplete: |
|
213
|
|
|
# Happens on non-debug build when a rule is "documentation-incomplete" |
|
214
|
|
|
continue |
|
215
|
|
|
if rule.template is None: |
|
216
|
|
|
# rule is not templated, skipping |
|
217
|
|
|
continue |
|
218
|
|
|
langs_to_generate = self.get_langs_to_generate(rule) |
|
219
|
|
|
self.build_rule( |
|
220
|
|
|
rule.id_, rule.title, rule.template, langs_to_generate) |
|
221
|
|
|
|
|
222
|
|
|
def build(self): |
|
223
|
|
|
""" |
|
224
|
|
|
Builds all templated content for all languages, writing |
|
225
|
|
|
the output to the correct build directories. |
|
226
|
|
|
""" |
|
227
|
|
|
|
|
228
|
|
|
for dir_ in self.output_dirs.values(): |
|
229
|
|
|
if not os.path.exists(dir_): |
|
230
|
|
|
os.makedirs(dir_) |
|
231
|
|
|
|
|
232
|
|
|
self.build_extra_ovals() |
|
233
|
|
|
self.build_all_rules() |
|
234
|
|
|
|
ComplianceAsCode /
content
