|
1
|
2 |
|
from __future__ import absolute_import |
|
2
|
2 |
|
from __future__ import print_function |
|
3
|
|
|
|
|
4
|
2 |
|
import codecs |
|
5
|
2 |
|
import yaml |
|
6
|
2 |
|
import sys |
|
7
|
2 |
|
import re |
|
8
|
|
|
|
|
9
|
2 |
|
from collections import OrderedDict |
|
10
|
|
|
|
|
11
|
2 |
|
from .jinja import extract_substitutions_dict_from_template, process_file |
|
12
|
2 |
|
from .constants import (PKG_MANAGER_TO_SYSTEM, |
|
13
|
|
|
PKG_MANAGER_TO_CONFIG_FILE, |
|
14
|
|
|
JINJA_MACROS_BASE_DEFINITIONS, |
|
15
|
|
|
JINJA_MACROS_HIGHLEVEL_DEFINITIONS) |
|
16
|
2 |
|
from .constants import DEFAULT_UID_MIN |
|
17
|
|
|
|
|
18
|
2 |
|
try: |
|
19
|
2 |
|
from yaml import CSafeLoader as yaml_SafeLoader |
|
20
|
2 |
|
except ImportError: |
|
21
|
2 |
|
from yaml import SafeLoader as yaml_SafeLoader |
|
22
|
|
|
|
|
23
|
|
|
|
|
24
|
2 |
|
def _bool_constructor(self, node): |
|
25
|
2 |
|
return self.construct_scalar(node) |
|
26
|
|
|
|
|
27
|
|
|
|
|
28
|
|
|
# Don't follow python bool case |
|
29
|
2 |
|
yaml_SafeLoader.add_constructor(u'tag:yaml.org,2002:bool', _bool_constructor) |
|
30
|
|
|
|
|
31
|
|
|
|
|
32
|
2 |
|
def _save_rename(result, stem, prefix): |
|
33
|
|
|
result["{0}_{1}".format(prefix, stem)] = stem |
|
34
|
|
|
|
|
35
|
|
|
|
|
36
|
2 |
|
def _open_yaml(stream, original_file=None, substitutions_dict={}): |
|
37
|
|
|
""" |
|
38
|
|
|
Open given file-like object and parse it as YAML. |
|
39
|
|
|
|
|
40
|
|
|
Optionally, pass the path to the original_file for better error handling |
|
41
|
|
|
when the file contents are passed. |
|
42
|
|
|
|
|
43
|
|
|
Return None if it contains "documentation_complete" key set to "false". |
|
44
|
|
|
""" |
|
45
|
2 |
|
try: |
|
46
|
2 |
|
yaml_contents = yaml.load(stream, Loader=yaml_SafeLoader) |
|
47
|
|
|
|
|
48
|
2 |
|
if yaml_contents.pop("documentation_complete", "true") == "false" and \ |
|
49
|
|
|
substitutions_dict.get("cmake_build_type") != "Debug": |
|
50
|
|
|
return None |
|
51
|
|
|
|
|
52
|
2 |
|
return yaml_contents |
|
53
|
|
|
except Exception as e: |
|
54
|
|
|
count = 0 |
|
55
|
|
|
_file = original_file |
|
56
|
|
|
if not _file: |
|
57
|
|
|
_file = stream |
|
58
|
|
|
with open(_file, "r") as e_file: |
|
59
|
|
|
lines = e_file.readlines() |
|
60
|
|
|
for line in lines: |
|
61
|
|
|
count = count + 1 |
|
62
|
|
|
if re.match(r"^\s*\t+\s*", line): |
|
63
|
|
|
print("Exception while handling file: %s" % _file, file=sys.stderr) |
|
64
|
|
|
print("TabIndentationError: Line %s contains tabs instead of spaces:" % (count), file=sys.stderr) |
|
65
|
|
|
print("%s\n\n" % repr(line.strip("\n")), file=sys.stderr) |
|
66
|
|
|
sys.exit(1) |
|
67
|
|
|
|
|
68
|
|
|
print("Exception while handling file: %s" % _file, file=sys.stderr) |
|
69
|
|
|
raise e |
|
70
|
|
|
|
|
71
|
|
|
|
|
72
|
2 |
|
def _get_implied_properties(existing_properties): |
|
73
|
|
|
result = dict() |
|
74
|
|
|
if "pkg_manager" in existing_properties: |
|
75
|
|
|
pkg_manager = existing_properties["pkg_manager"] |
|
76
|
|
|
if "pkg_system" not in existing_properties: |
|
77
|
|
|
result["pkg_system"] = PKG_MANAGER_TO_SYSTEM[pkg_manager] |
|
78
|
|
|
if "pkg_manager_config_file" not in existing_properties: |
|
79
|
|
|
if pkg_manager in PKG_MANAGER_TO_CONFIG_FILE: |
|
80
|
|
|
result["pkg_manager_config_file"] = PKG_MANAGER_TO_CONFIG_FILE[pkg_manager] |
|
81
|
|
|
|
|
82
|
|
|
if "uid_min" not in existing_properties: |
|
83
|
|
|
result["uid_min"] = DEFAULT_UID_MIN |
|
84
|
|
|
|
|
85
|
|
|
if "auid" not in existing_properties: |
|
86
|
|
|
result["auid"] = existing_properties.get("uid_min", DEFAULT_UID_MIN) |
|
87
|
|
|
|
|
88
|
|
|
return result |
|
89
|
|
|
|
|
90
|
|
|
|
|
91
|
2 |
|
def open_and_expand(yaml_file, substitutions_dict=None): |
|
92
|
|
|
""" |
|
93
|
|
|
Process the file as a template, using substitutions_dict to perform |
|
94
|
|
|
expansion. Then, process the expansion result as a YAML content. |
|
95
|
|
|
|
|
96
|
|
|
See also: _open_yaml |
|
97
|
|
|
""" |
|
98
|
2 |
|
if substitutions_dict is None: |
|
99
|
2 |
|
substitutions_dict = dict() |
|
100
|
|
|
|
|
101
|
2 |
|
expanded_template = process_file(yaml_file, substitutions_dict) |
|
102
|
2 |
|
yaml_contents = _open_yaml(expanded_template, yaml_file, substitutions_dict) |
|
103
|
2 |
|
return yaml_contents |
|
104
|
|
|
|
|
105
|
|
|
|
|
106
|
2 |
|
def open_and_macro_expand(yaml_file, substitutions_dict=None): |
|
107
|
|
|
""" |
|
108
|
|
|
Do the same as open_and_expand, but load definitions of macros |
|
109
|
|
|
so they can be expanded in the template. |
|
110
|
|
|
""" |
|
111
|
2 |
|
if substitutions_dict is None: |
|
112
|
2 |
|
substitutions_dict = dict() |
|
113
|
|
|
|
|
114
|
2 |
|
try: |
|
115
|
2 |
|
macro_definitions = extract_substitutions_dict_from_template( |
|
116
|
|
|
JINJA_MACROS_BASE_DEFINITIONS, substitutions_dict) |
|
117
|
2 |
|
macro_definitions.update(extract_substitutions_dict_from_template( |
|
118
|
|
|
JINJA_MACROS_HIGHLEVEL_DEFINITIONS, substitutions_dict)) |
|
119
|
|
|
except Exception as exc: |
|
120
|
|
|
msg = ("Error extracting macro definitions: {0}" |
|
121
|
|
|
.format(str(exc))) |
|
122
|
|
|
raise RuntimeError(msg) |
|
123
|
2 |
|
substitutions_dict.update(macro_definitions) |
|
124
|
2 |
|
return open_and_expand(yaml_file, substitutions_dict) |
|
125
|
|
|
|
|
126
|
|
|
|
|
127
|
2 |
|
def open_raw(yaml_file): |
|
128
|
|
|
""" |
|
129
|
|
|
Open given file-like object and parse it as YAML |
|
130
|
|
|
without performing any kind of template processing |
|
131
|
|
|
|
|
132
|
|
|
See also: _open_yaml |
|
133
|
|
|
""" |
|
134
|
2 |
|
with codecs.open(yaml_file, "r", "utf8") as stream: |
|
135
|
2 |
|
yaml_contents = _open_yaml(stream, original_file=yaml_file) |
|
136
|
2 |
|
return yaml_contents |
|
137
|
|
|
|
|
138
|
|
|
|
|
139
|
2 |
|
def open_environment(build_config_yaml, product_yaml): |
|
140
|
|
|
contents = open_raw(build_config_yaml) |
|
141
|
|
|
contents.update(open_raw(product_yaml)) |
|
142
|
|
|
contents.update(_get_implied_properties(contents)) |
|
143
|
|
|
return contents |
|
144
|
|
|
|
|
145
|
|
|
|
|
146
|
2 |
|
def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict): |
|
147
|
|
|
""" |
|
148
|
|
|
Drop-in replacement for yaml.load(), but preserves order of dictionaries |
|
149
|
|
|
""" |
|
150
|
2 |
|
class OrderedLoader(Loader): |
|
151
|
2 |
|
pass |
|
152
|
|
|
|
|
153
|
2 |
|
def construct_mapping(loader, node): |
|
154
|
2 |
|
loader.flatten_mapping(node) |
|
155
|
2 |
|
return object_pairs_hook(loader.construct_pairs(node)) |
|
156
|
2 |
|
OrderedLoader.add_constructor( |
|
157
|
|
|
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, |
|
158
|
|
|
construct_mapping) |
|
159
|
2 |
|
return yaml.load(stream, OrderedLoader) |
|
160
|
|
|
|
|
161
|
|
|
|
|
162
|
2 |
|
def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): |
|
163
|
|
|
""" |
|
164
|
|
|
Drop-in replacement for yaml.dump(), but preserves order of dictionaries |
|
165
|
|
|
""" |
|
166
|
2 |
|
class OrderedDumper(Dumper): |
|
167
|
|
|
# fix tag indentations |
|
168
|
2 |
|
def increase_indent(self, flow=False, indentless=False): |
|
169
|
2 |
|
return super(OrderedDumper, self).increase_indent(flow, False) |
|
170
|
|
|
|
|
171
|
2 |
|
def _dict_representer(dumper, data): |
|
172
|
2 |
|
return dumper.represent_mapping( |
|
173
|
|
|
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, |
|
174
|
|
|
data.items()) |
|
175
|
|
|
|
|
176
|
2 |
|
OrderedDumper.add_representer(OrderedDict, _dict_representer) |
|
177
|
|
|
|
|
178
|
|
|
# Fix formatting by adding a space in between tasks |
|
179
|
2 |
|
unformatted_yaml = yaml.dump(data, None, OrderedDumper, **kwds) |
|
180
|
2 |
|
formatted_yaml = re.sub(r"[\n]+([\s]*)- name", r"\n\n\1- name", unformatted_yaml) |
|
181
|
|
|
|
|
182
|
2 |
|
if stream is not None: |
|
183
|
2 |
|
return stream.write(formatted_yaml) |
|
184
|
|
|
else: |
|
185
|
|
|
return formatted_yaml |
|
186
|
|
|
|
ComplianceAsCode /
content
