_fetch_and_compile_index() - Code Metrics - Inspection of "Pack management" - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#2842)

by Edward

created 2016-10-19 03:45 UTC

_fetch_and_compile_index() C

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	7
dl	0
loc	44
rs	5.5
c	1
b	0
f	0

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools
import json

import requests
import six
from oslo_config import cfg

from st2common import log as logging
from st2common.models.api.pack import PackAPI
from st2common.persistence.pack import Pack

__all__ = [
    'get_pack_by_ref',
    'fetch_pack_index',
    'get_pack_from_index',
    'search_pack_index',
    'check_index_health'
]

EXCLUDE_FIELDS = [
    "repo_url",
    "email"
]

SEARCH_PRIORITY = [
    "name",
    "keywords"
]

LOG = logging.getLogger(__name__)


def _build_index_list(index_url):
    if not index_url:
        # Reversing the indexes list from config so that the indexes have
        # descending (left-to-right) priority.
        # When multiple indexes have a pack with a given name, the index
        # that comes first in the list will be used.
        index_urls = cfg.CONF.content.index_url[::-1]
    elif isinstance(index_url, str):
        index_urls = [index_url]
    elif hasattr(index_url, '__iter__'):
        index_urls = index_url
    else:
        raise TypeError('"index_url" should either be a string or an iterable object.')
    return index_urls


def _fetch_and_compile_index(index_urls, logger=None):
    """
    Go through the index list and compile results into a single object.
    """
    status = []
    index = {}

    for index_url in index_urls:

        index_status = {
            'url': index_url,
            'packs': 0,
            'message': None,
            'error': None,
        }

        try:
            request = requests.get(index_url)
            request.raise_for_status()
            index_json = request.json()
        except ValueError as e:
            index_status['error'] = 'malformed'
            index_status['message'] = e
        except requests.exceptions.RequestException as e:
            index_status['error'] = 'unresponsive'
            index_status['message'] = e

        if not index_json:
            index_status['error'] = 'empty'
            index_status['message'] = 'The index URL returned an empty object.'
        elif type(index_json) is list:
            index_status['error'] = 'malformed'
            index_status['message'] = 'Expected an index object, got a list instead.'

        if index_status['error']:
            logger.error("Index parsing error: %s" % json.dumps(index_status, indent=4))
        else:
            index_status['message'] = 'Success.'
            index_status['packs'] = len(index_json)
            index.update(index_json)

        status.append(index_status)

    return index, status


def get_pack_by_ref(pack_ref):
    """
    Retrieve PackDB by the provided reference.
    """
    pack_db = Pack.get_by_ref(pack_ref)
    return pack_db


def fetch_pack_index(index_url=None, logger=None):
    """
    Fetch the pack indexes (either from the config or provided as an argument)
    and return the object.
    """
    logger = logger or LOG

    index_urls = _build_index_list(index_url)
    index, status = _fetch_and_compile_index(index_urls, logger)

    # If one of the indexes on the list is unresponsive, we do not throw
    # immediately. The only case where an exception is raised is when no
    # results could be obtained from all listed indexes.
    # This behavior allows for mirrors / backups and handling connection
    # or network issues in one of the indexes.
    if not index:
        raise ValueError("No results from the %s: tried %s.\nStatus: %s" % (
            ("index" if len(index_urls) == 1 else "indexes"),
            ", ".join(index_urls),
            json.dumps(status, indent=4)
        ))
    return index


def check_index_health(index_url=None, status=None, logger=None):
    """
    Check if all listed indexes are healthy: they should be reachable,
    return valid JSON objects, and yield more than one result.
    """
    logger = logger or LOG

    if not status:
        index_urls = _build_index_list(index_url)
        _, status = _fetch_and_compile_index(index_urls, logger)

    health = {
        "indexes": {
            "count": len(status),
            "valid": 0,
            "invalid": 0,
            "errors": {},
            "status": status,
        },
        "packs": {
            "count": 0,
        },
    }

    for index in status:
        if index['error']:
            error_count = health['indexes']['errors'].get(index['error'], 0) + 1
            health['indexes']['invalid'] += 1
            health['indexes']['errors'][index['error']] = error_count
        else:
            health['indexes']['valid'] += 1
        health['packs']['count'] += index['packs']

    return health


def get_pack_from_index(pack):
    """
    Search index by pack name.
    Returns a pack.
    """
    if not pack:
        raise ValueError("Pack name must be specified.")

    index = fetch_pack_index()

    return PackAPI(**index.get(pack))


def search_pack_index(query, exclude=None, priority=None):
    """
    Search the pack index by query.
    Returns a list of matches for a query.
    """
    if not query:
        raise ValueError("Query must be specified.")

    if not exclude:
        exclude = EXCLUDE_FIELDS
    if not priority:
        priority = SEARCH_PRIORITY

    index = fetch_pack_index()

    matches = [[] for _ in range(len(priority) + 1)]
    for pack_dict in six.itervalues(index):
        pack = PackAPI(**pack_dict)

        for key, value in six.iteritems(vars(pack)):
            if not hasattr(value, '__contains__'):
                value = str(value)

            if key not in exclude and query in value:
                if key in priority:
                    matches[priority.index(key)].append(pack)
                else:
                    matches[-1].append(pack)
                break

    return list(itertools.chain.from_iterable(matches))


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import itertools
17			import json
18
19			import requests
20			import six
21			from oslo_config import cfg
22
23			from st2common import log as logging
24			from st2common.models.api.pack import PackAPI
25			from st2common.persistence.pack import Pack
26
27			__all__ = [
28			'get_pack_by_ref',
29			'fetch_pack_index',
30			'get_pack_from_index',
31			'search_pack_index',
32			'check_index_health'
33			]
34
35			EXCLUDE_FIELDS = [
36			"repo_url",
37			"email"
38			]
39
40			SEARCH_PRIORITY = [
41			"name",
42			"keywords"
43			]
44
45			LOG = logging.getLogger(__name__)
46
47
48			def _build_index_list(index_url):
49			if not index_url:
50			# Reversing the indexes list from config so that the indexes have
51			# descending (left-to-right) priority.
52			# When multiple indexes have a pack with a given name, the index
53			# that comes first in the list will be used.
54			index_urls = cfg.CONF.content.index_url[::-1]
55			elif isinstance(index_url, str):
56			index_urls = [index_url]
57			elif hasattr(index_url, '__iter__'):
58			index_urls = index_url
59			else:
60			raise TypeError('"index_url" should either be a string or an iterable object.')
61			return index_urls
62
63
64			def _fetch_and_compile_index(index_urls, logger=None):
65			"""
66			Go through the index list and compile results into a single object.
67			"""
68			status = []
69			index = {}
70
71			for index_url in index_urls:
72
73			index_status = {
74			'url': index_url,
75			'packs': 0,
76			'message': None,
77			'error': None,
78			}
79
80			try:
81			request = requests.get(index_url)
82			request.raise_for_status()
83			index_json = request.json()
84			except ValueError as e:
85			index_status['error'] = 'malformed'
86			index_status['message'] = e
87			except requests.exceptions.RequestException as e:
88			index_status['error'] = 'unresponsive'
89			index_status['message'] = e
90
91			if not index_json:
92			index_status['error'] = 'empty'
93			index_status['message'] = 'The index URL returned an empty object.'
94			elif type(index_json) is list:
95			index_status['error'] = 'malformed'
96			index_status['message'] = 'Expected an index object, got a list instead.'
97
98			if index_status['error']:
99			logger.error("Index parsing error: %s" % json.dumps(index_status, indent=4))
100			else:
101			index_status['message'] = 'Success.'
102			index_status['packs'] = len(index_json)
103			index.update(index_json)
104
105			status.append(index_status)
106
107			return index, status
108
109
110			def get_pack_by_ref(pack_ref):
111			"""
112			Retrieve PackDB by the provided reference.
113			"""
114			pack_db = Pack.get_by_ref(pack_ref)
115			return pack_db
116
117
118			def fetch_pack_index(index_url=None, logger=None):
119			"""
120			Fetch the pack indexes (either from the config or provided as an argument)
121			and return the object.
122			"""
123			logger = logger or LOG
124
125			index_urls = _build_index_list(index_url)
126			index, status = _fetch_and_compile_index(index_urls, logger)
127
128			# If one of the indexes on the list is unresponsive, we do not throw
129			# immediately. The only case where an exception is raised is when no
130			# results could be obtained from all listed indexes.
131			# This behavior allows for mirrors / backups and handling connection
132			# or network issues in one of the indexes.
133			if not index:
134			raise ValueError("No results from the %s: tried %s.\nStatus: %s" % (
135			("index" if len(index_urls) == 1 else "indexes"),
136			", ".join(index_urls),
137			json.dumps(status, indent=4)
138			))
139			return index
140
141
142			def check_index_health(index_url=None, status=None, logger=None):
143			"""
144			Check if all listed indexes are healthy: they should be reachable,
145			return valid JSON objects, and yield more than one result.
146			"""
147			logger = logger or LOG
148
149			if not status:
150			index_urls = _build_index_list(index_url)
151			_, status = _fetch_and_compile_index(index_urls, logger)
152
153			health = {
154			"indexes": {
155			"count": len(status),
156			"valid": 0,
157			"invalid": 0,
158			"errors": {},
159			"status": status,
160			},
161			"packs": {
162			"count": 0,
163			},
164			}
165
166			for index in status:
167			if index['error']:
168			error_count = health['indexes']['errors'].get(index['error'], 0) + 1
169			health['indexes']['invalid'] += 1
170			health['indexes']['errors'][index['error']] = error_count
171			else:
172			health['indexes']['valid'] += 1
173			health['packs']['count'] += index['packs']
174
175			return health
176
177
178			def get_pack_from_index(pack):
179			"""
180			Search index by pack name.
181			Returns a pack.
182			"""
183			if not pack:
184			raise ValueError("Pack name must be specified.")
185
186			index = fetch_pack_index()
187
188			return PackAPI(**index.get(pack))
189
190
191			def search_pack_index(query, exclude=None, priority=None):
192			"""
193			Search the pack index by query.
194			Returns a list of matches for a query.
195			"""
196			if not query:
197			raise ValueError("Query must be specified.")
198
199			if not exclude:
200			exclude = EXCLUDE_FIELDS
201			if not priority:
202			priority = SEARCH_PRIORITY
203
204			index = fetch_pack_index()
205
206			matches = [[] for _ in range(len(priority) + 1)]
207			for pack_dict in six.itervalues(index):
208			pack = PackAPI(**pack_dict)
209
210			for key, value in six.iteritems(vars(pack)):
211			if not hasattr(value, '__contains__'):
212			value = str(value)
213
214			if key not in exclude and query in value:
215			if key in priority:
216			matches[priority.index(key)].append(pack)
217			else:
218			matches[-1].append(pack)
219			break
220
221			return list(itertools.chain.from_iterable(matches))
222

StackStorm / st2

Pull Request — master (#2842)

_fetch_and_compile_index() C

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like