ssg.templates.Template.__init__() - Code Metrics - Inspection of "Skip irrelevant template directories" - ComplianceAsCode/content - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#6432)

by Matěj

created 2020-11-26 17:11 UTC

ssg.templates.Template.init() A

↳ Parent: ssg.templates

Complexity

Conditions

Size

Total Lines	6
Code Lines	6

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	0
CRAP Score	2

Importance

Changes

Metric	Value
cc	1
eloc	6
nop	3
dl	0
loc	6
ccs	0
cts	6
cp	0
crap	2
rs	10
c	0
b	0
f	0

from __future__ import absolute_import
from __future__ import print_function

import os
import sys
import imp
import glob

import ssg.build_yaml
import ssg.utils
import ssg.yaml

try:
    from urllib.parse import quote
except ImportError:
    from urllib import quote

languages = ["anaconda", "ansible", "bash", "oval", "puppet", "ignition", "kubernetes"]
preprocessing_file_name = "template.py"
lang_to_ext_map = {
    "anaconda": ".anaconda",
    "ansible": ".yml",
    "bash": ".sh",
    "oval": ".xml",
    "puppet": ".pp",
    "ignition": ".yml",
    "kubernetes": ".yml"
}


templates = dict()


class Template():
    def __init__(self, template_root_directory, name):
        self.template_root_directory = template_root_directory
        self.name = name
        self.template_path = os.path.join(self.template_root_directory, self.name)
        self.template_yaml_path = os.path.join(self.template_path, "template.yml")
        self.preprocessing_file_path = os.path.join(self.template_path, preprocessing_file_name)

    def load(self):
        if not os.path.exists(self.preprocessing_file_path):
            self.preprocessing_file_path = None
        self.langs = []
        template_yaml = ssg.yaml.open_raw(self.template_yaml_path)
        for lang in template_yaml["supported_languages"]:
            if lang not in languages:
                raise ValueError("The template {0} declares to support the {1} language,"
                "but this language is not supported by the content.".format(self.name, lang))
            langfilename = lang + ".template"
            if not os.path.exists(os.path.join(self.template_path, langfilename)):
                raise ValueError("The template {0} declares to support the {1} language,"
                "but the implementation file is missing.".format(self.name, lang))
            self.langs.append(lang)

    def preprocess(self, parameters, lang):
        # if no template.py file exists, skip this preprocessing part
        if self.preprocessing_file_path is not None:
            module_name = "tmpmod" # dummy module name, we don't need it later
            preprocess_mod = imp.load_source(module_name, self.preprocessing_file_path)
            parameters = preprocess_mod.preprocess(parameters.copy(), lang)
        # TODO: Remove this right after the variables in templates are renamed
        # to lowercase
        uppercases = dict()
        for k, v in parameters.items():
            uppercases[k.upper()] = v
        return uppercases

    def looks_like_template(self):
        if not os.path.isdir(self.template_root_directory):
            return False
        if os.path.islink(self.template_root_directory):
            return False
        template_sources = glob.glob(os.path.join(self.template_path, "*.template"))
        if not os.path.isfile(self.template_yaml_path) and not template_sources:
            return False
        return True


class Builder(object):
    """
    Class for building all templated content for a given product.

    To generate content from templates, pass the env_yaml, path to the
    directory with resolved rule YAMLs, path to the directory that contains
    templates, path to the output directory for checks and a path to the
    output directory for remediations into the constructor. Then, call the
    method build() to perform a build.
    """
    def __init__(
            self, env_yaml, resolved_rules_dir, templates_dir,
            remediations_dir, checks_dir):
        self.env_yaml = env_yaml
        self.resolved_rules_dir = resolved_rules_dir
        self.templates_dir = templates_dir
        self.remediations_dir = remediations_dir
        self.checks_dir = checks_dir
        self.output_dirs = dict()
        for lang in languages:
            if lang == "oval":
                # OVAL checks need to be put to a different directory because
                # they are processed differently than remediations later in the
                # build process
                output_dir = self.checks_dir
            else:
                output_dir = self.remediations_dir
            dir_ = os.path.join(output_dir, lang)
            self.output_dirs[lang] = dir_
        # scan directory structure and dynamically create list of templates
        for item in os.listdir(self.templates_dir):
            itempath = os.path.join(self.templates_dir, item)
            maybe_template = Template(templates_dir, item)
            if maybe_template.looks_like_template():
                maybe_template.load()
                templates[item] = maybe_template


    def build_lang(
            self, rule_id, template_name, template_vars, lang, local_env_yaml):
        """
        Builds templated content for a given rule for a given language.
        Writes the output to the correct build directories.
        """
        if lang not in templates[template_name].langs:
            return
        template_file_name = lang + ".template"
        template_file_path = os.path.join(self.templates_dir, template_name, template_file_name)
        ext = lang_to_ext_map[lang]
        output_file_name = rule_id + ext
        output_filepath = os.path.join(
            self.output_dirs[lang], output_file_name)
        template_parameters = templates[template_name].preprocess(template_vars, lang)
        jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)
        filled_template = ssg.jinja.process_file_with_macros(
            template_file_path, jinja_dict)
        with open(output_filepath, "w") as f:
            f.write(filled_template)

    def get_langs_to_generate(self, rule):
        """
        For a given rule returns list of languages that should be generated
        from templates. This is controlled by "template_backends" in rule.yml.
        """
        if "backends" in rule.template:
            backends = rule.template["backends"]
            for lang in backends:
                if lang not in languages:
                    raise RuntimeError(
                        "Rule {0} wants to generate unknown language '{1}"
                        "from a template.".format(rule.id_, lang)
                    )
            langs_to_generate = []
            for lang in languages:
                backend = backends.get(lang, "on")
                if backend == "on":
                    langs_to_generate.append(lang)
            return langs_to_generate
        else:
            return languages

    def build_rule(self, rule_id, rule_title, template, langs_to_generate):
        """
        Builds templated content for a given rule for selected languages,
        writing the output to the correct build directories.
        """
        try:
            template_name = template["name"]
        except KeyError:
            raise ValueError(
                "Rule {0} is missing template name under template key".format(
                    rule_id))
        if template_name not in templates.keys():
            raise ValueError(
                "Rule {0} uses template {1} which does not exist.".format(
                    rule_id, template_name))
        try:
            template_vars = template["vars"]
        except KeyError:
            raise ValueError(
                "Rule {0} does not contain mandatory 'vars:' key under "
                "'template:' key.".format(rule_id))
        # Add the rule ID which will be reused in OVAL templates as OVAL
        # definition ID so that the build system matches the generated
        # check with the rule.
        template_vars["_rule_id"] = rule_id
        # checks and remediations are processed with a custom YAML dict
        local_env_yaml = self.env_yaml.copy()
        local_env_yaml["rule_id"] = rule_id
        local_env_yaml["rule_title"] = rule_title
        local_env_yaml["products"] = self.env_yaml["product"]
        for lang in langs_to_generate:
            self.build_lang(
                rule_id, template_name, template_vars, lang, local_env_yaml)

    def build_extra_ovals(self):
        declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml")
        declaration = ssg.yaml.open_raw(declaration_path)
        for oval_def_id, template in declaration.items():
            langs_to_generate = ["oval"]
            # Since OVAL definition ID in shorthand format is always the same
            # as rule ID, we can use it instead of the rule ID even if no rule
            # with that ID exists
            self.build_rule(
                oval_def_id, oval_def_id, template, langs_to_generate)

    def build_all_rules(self):
        for rule_file in os.listdir(self.resolved_rules_dir):
            rule_path = os.path.join(self.resolved_rules_dir, rule_file)
            try:
                rule = ssg.build_yaml.Rule.from_yaml(rule_path, self.env_yaml)
            except ssg.build_yaml.DocumentationNotComplete:
                # Happens on non-debug build when a rule is "documentation-incomplete"
                continue
            if rule.template is None:
                # rule is not templated, skipping
                continue
            langs_to_generate = self.get_langs_to_generate(rule)
            self.build_rule(
                rule.id_, rule.title, rule.template, langs_to_generate)

    def build(self):
        """
        Builds all templated content for all languages, writing
        the output to the correct build directories.
        """

        for dir_ in self.output_dirs.values():
            if not os.path.exists(dir_):
                os.makedirs(dir_)

        self.build_extra_ovals()
        self.build_all_rules()


1			from __future__ import absolute_import
2			from __future__ import print_function
3
4			import os
5			import sys
6			import imp
7			import glob
8
9			import ssg.build_yaml
10			import ssg.utils
11			import ssg.yaml
12
13			try:
14			from urllib.parse import quote
15			except ImportError:
16			from urllib import quote
17
18			languages = ["anaconda", "ansible", "bash", "oval", "puppet", "ignition", "kubernetes"]
19			preprocessing_file_name = "template.py"
20			lang_to_ext_map = {
21			"anaconda": ".anaconda",
22			"ansible": ".yml",
23			"bash": ".sh",
24			"oval": ".xml",
25			"puppet": ".pp",
26			"ignition": ".yml",
27			"kubernetes": ".yml"
28			}
29
30
31			templates = dict()
32
33
34			class Template():
35			def __init__(self, template_root_directory, name):
36			self.template_root_directory = template_root_directory
37			self.name = name
38			self.template_path = os.path.join(self.template_root_directory, self.name)
39			self.template_yaml_path = os.path.join(self.template_path, "template.yml")
40			self.preprocessing_file_path = os.path.join(self.template_path, preprocessing_file_name)
41
42			def load(self):
43			if not os.path.exists(self.preprocessing_file_path):
44			self.preprocessing_file_path = None
45			self.langs = []
46			template_yaml = ssg.yaml.open_raw(self.template_yaml_path)
47			for lang in template_yaml["supported_languages"]:
48			if lang not in languages:
49			raise ValueError("The template {0} declares to support the {1} language,"
50			"but this language is not supported by the content.".format(self.name, lang))
51			langfilename = lang + ".template"
52			if not os.path.exists(os.path.join(self.template_path, langfilename)):
53			raise ValueError("The template {0} declares to support the {1} language,"
54			"but the implementation file is missing.".format(self.name, lang))
55			self.langs.append(lang)
56
57			def preprocess(self, parameters, lang):
58			# if no template.py file exists, skip this preprocessing part
59			if self.preprocessing_file_path is not None:
60			module_name = "tmpmod" # dummy module name, we don't need it later
61			preprocess_mod = imp.load_source(module_name, self.preprocessing_file_path)
62			parameters = preprocess_mod.preprocess(parameters.copy(), lang)
63			# TODO: Remove this right after the variables in templates are renamed
64			# to lowercase
65			uppercases = dict()
66			for k, v in parameters.items():
67			uppercases[k.upper()] = v
68			return uppercases
69
70			def looks_like_template(self):
71			if not os.path.isdir(self.template_root_directory):
72			return False
73			if os.path.islink(self.template_root_directory):
74			return False
75			template_sources = glob.glob(os.path.join(self.template_path, "*.template"))
76			if not os.path.isfile(self.template_yaml_path) and not template_sources:
77			return False
78			return True
79
80
81			class Builder(object):
82			"""
83			Class for building all templated content for a given product.
84
85			To generate content from templates, pass the env_yaml, path to the
86			directory with resolved rule YAMLs, path to the directory that contains
87			templates, path to the output directory for checks and a path to the
88			output directory for remediations into the constructor. Then, call the
89			method build() to perform a build.
90			"""
91			def __init__(
92			self, env_yaml, resolved_rules_dir, templates_dir,
93			remediations_dir, checks_dir):
94			self.env_yaml = env_yaml
95			self.resolved_rules_dir = resolved_rules_dir
96			self.templates_dir = templates_dir
97			self.remediations_dir = remediations_dir
98			self.checks_dir = checks_dir
99			self.output_dirs = dict()
100			for lang in languages:
101			if lang == "oval":
102			# OVAL checks need to be put to a different directory because
103			# they are processed differently than remediations later in the
104			# build process
105			output_dir = self.checks_dir
106			else:
107			output_dir = self.remediations_dir
108			dir_ = os.path.join(output_dir, lang)
109			self.output_dirs[lang] = dir_
110			# scan directory structure and dynamically create list of templates
111			for item in os.listdir(self.templates_dir):
112			itempath = os.path.join(self.templates_dir, item)
113			maybe_template = Template(templates_dir, item)
114			if maybe_template.looks_like_template():
115			maybe_template.load()
116			templates[item] = maybe_template
117
118
119			def build_lang(
120			self, rule_id, template_name, template_vars, lang, local_env_yaml):
121			"""
122			Builds templated content for a given rule for a given language.
123			Writes the output to the correct build directories.
124			"""
125			if lang not in templates[template_name].langs:
126			return
127			template_file_name = lang + ".template"
128			template_file_path = os.path.join(self.templates_dir, template_name, template_file_name)
129			ext = lang_to_ext_map[lang]
130			output_file_name = rule_id + ext
131			output_filepath = os.path.join(
132			self.output_dirs[lang], output_file_name)
133			template_parameters = templates[template_name].preprocess(template_vars, lang)
134			jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)
135			filled_template = ssg.jinja.process_file_with_macros(
136			template_file_path, jinja_dict)
137			with open(output_filepath, "w") as f:
138			f.write(filled_template)
139
140			def get_langs_to_generate(self, rule):
141			"""
142			For a given rule returns list of languages that should be generated
143			from templates. This is controlled by "template_backends" in rule.yml.
144			"""
145			if "backends" in rule.template:
146			backends = rule.template["backends"]
147			for lang in backends:
148			if lang not in languages:
149			raise RuntimeError(
150			"Rule {0} wants to generate unknown language '{1}"
151			"from a template.".format(rule.id_, lang)
152			)
153			langs_to_generate = []
154			for lang in languages:
155			backend = backends.get(lang, "on")
156			if backend == "on":
157			langs_to_generate.append(lang)
158			return langs_to_generate
159			else:
160			return languages
161
162			def build_rule(self, rule_id, rule_title, template, langs_to_generate):
163			"""
164			Builds templated content for a given rule for selected languages,
165			writing the output to the correct build directories.
166			"""
167			try:
168			template_name = template["name"]
169			except KeyError:
170			raise ValueError(
171			"Rule {0} is missing template name under template key".format(
172			rule_id))
173			if template_name not in templates.keys():
174			raise ValueError(
175			"Rule {0} uses template {1} which does not exist.".format(
176			rule_id, template_name))
177			try:
178			template_vars = template["vars"]
179			except KeyError:
180			raise ValueError(
181			"Rule {0} does not contain mandatory 'vars:' key under "
182			"'template:' key.".format(rule_id))
183			# Add the rule ID which will be reused in OVAL templates as OVAL
184			# definition ID so that the build system matches the generated
185			# check with the rule.
186			template_vars["_rule_id"] = rule_id
187			# checks and remediations are processed with a custom YAML dict
188			local_env_yaml = self.env_yaml.copy()
189			local_env_yaml["rule_id"] = rule_id
190			local_env_yaml["rule_title"] = rule_title
191			local_env_yaml["products"] = self.env_yaml["product"]
192			for lang in langs_to_generate:
193			self.build_lang(
194			rule_id, template_name, template_vars, lang, local_env_yaml)
195
196			def build_extra_ovals(self):
197			declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml")
198			declaration = ssg.yaml.open_raw(declaration_path)
199			for oval_def_id, template in declaration.items():
200			langs_to_generate = ["oval"]
201			# Since OVAL definition ID in shorthand format is always the same
202			# as rule ID, we can use it instead of the rule ID even if no rule
203			# with that ID exists
204			self.build_rule(
205			oval_def_id, oval_def_id, template, langs_to_generate)
206
207			def build_all_rules(self):
208			for rule_file in os.listdir(self.resolved_rules_dir):
209			rule_path = os.path.join(self.resolved_rules_dir, rule_file)
210			try:
211			rule = ssg.build_yaml.Rule.from_yaml(rule_path, self.env_yaml)
212			except ssg.build_yaml.DocumentationNotComplete:
213			# Happens on non-debug build when a rule is "documentation-incomplete"
214			continue
215			if rule.template is None:
216			# rule is not templated, skipping
217			continue
218			langs_to_generate = self.get_langs_to_generate(rule)
219			self.build_rule(
220			rule.id_, rule.title, rule.template, langs_to_generate)
221
222			def build(self):
223			"""
224			Builds all templated content for all languages, writing
225			the output to the correct build directories.
226			"""
227
228			for dir_ in self.output_dirs.values():
229			if not os.path.exists(dir_):
230			os.makedirs(dir_)
231
232			self.build_extra_ovals()
233			self.build_all_rules()
234

ComplianceAsCode / content

Pull Request — master (#6432)

ssg.templates.Template.__init__() A

Complexity

Size

Duplication

Code Coverage

Importance

Duplication Side-by-Side

Filter issues like

ssg.templates.Template.init() A