buildtimetrend.send_build_data_service() - Code Metrics - Inspection of "increase build number" - buildtimetrend/python-lib - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( fec650...1a3e44 )

by Dieter

created 2016-01-24 13:25 UTC

buildtimetrend.send_build_data_service() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	4
dl	0
loc	22
rs	8.9197

# vim: set expandtab sw=4 ts=4:
"""
Interface to Keen IO.

Copyright (C) 2014-2016 Dieter Adriaenssens <[email protected]>

This file is part of buildtimetrend/python-lib
<https://github.com/buildtimetrend/python-lib/>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""

from __future__ import division
from builtins import str
import os
from buildtimetrend import logger
import copy
import keen
import requests
from keen import scoped_keys
from buildtimetrend.settings import Settings
from buildtimetrend.tools import check_dict
from buildtimetrend.tools import is_list
from buildtimetrend.tools import is_string
from buildtimetrend.buildjob import BuildJob


TIME_INTERVALS = {
    'week': {'name': 'week', 'timeframe': 'this_7_days', 'max_age': 600},
    'month': {'name': 'month', 'timeframe': 'this_30_days', 'max_age': 600},
    'year': {'name': 'year', 'timeframe': 'this_52_weeks', 'max_age': 1800}
}
KEEN_PROJECT_INFO_NAME = "buildtime_trend"


def keen_has_project_id():
    """Check if Keen.io project ID is set."""
    if "KEEN_PROJECT_ID" in os.environ or keen.project_id is not None:
        return True

    logger.warning("Keen.io Project ID is not set")
    return False


def keen_has_master_key():
    """Check if Keen.io Master API key is set."""
    if "KEEN_MASTER_KEY" in os.environ or keen.master_key is not None:
        return True

    logger.warning("Keen.io Master API Key is not set")
    return False


def keen_has_write_key():
    """Check if Keen.io Write Key is set."""
    if "KEEN_WRITE_KEY" in os.environ or keen.write_key is not None:
        return True

    logger.warning("Keen.io Write Key is not set")
    return False


def keen_has_read_key():
    """Check if Keen.io Read key is set."""
    if "KEEN_READ_KEY" in os.environ or keen.read_key is not None:
        return True

    logger.warning("Keen.io Read Key is not set")
    return False


def keen_is_writable():
    """Check if login keys for Keen IO API are set, to allow writing."""
    if keen_has_project_id() and keen_has_write_key():
        return True

    logger.warning("Keen.io Write Key is not set")
    return False


def keen_is_readable():
    """Check if login keys for Keen IO API are set, to allow reading."""
    if keen_has_project_id() and keen_has_read_key():
        return True

    logger.warning("Keen.io Read Key is not set")
    return False


def keen_io_generate_read_key(repo):
    """
    Create scoped key for reading only the build-stages related data.

    Param repo : github repository slug (fe. buildtimetrend/python-lib)
    """
    if not keen_has_master_key():
        logger.warning("Keen.io Read Key was not created,"
                       " keen.master_key is not defined.")
        return None

    master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")

    privileges = {
        "allowed_operations": ["read"]
    }

    if repo is not None:
        privileges["filters"] = [get_repo_filter(repo)]

    logger.info("Keen.io Read Key is created for %s", repo)
    return scoped_keys.encrypt(master_key, privileges)


def keen_io_generate_write_key():
    """Create scoped key for write access to Keen.io database."""
    if not keen_has_master_key():
        logger.warning("Keen.io Write Key was not created,"
                       " keen.master_key is not defined.")
        return None

    master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")

    privileges = {
        "allowed_operations": ["write"]
    }

    logger.info("Keen.io Write Key is created")
    return scoped_keys.encrypt(master_key, privileges)


def send_build_data(buildjob, detail=None):
    """
    Send build data generated by client to keen.io.

    Parameters:
    - buildjob : BuildJob instance
    - detail : Data storage detail level :
               'minimal', 'basic', 'full', 'extended'
    """
    if not isinstance(buildjob, BuildJob):
        raise TypeError("param buildjob should be a BuildJob instance")

    data_detail = Settings().get_value_or_setting("data_detail", detail)

    if keen_is_writable():
        logger.info(
            "Sending client build job data to Keen.io (data detail: %s)",
            data_detail
        )
        keen_add_event("build_jobs", {"job": buildjob.to_dict()})
        if data_detail in ("full", "extended"):
            keen_add_events("build_stages", buildjob.stages_to_list())


def send_build_data_service(buildjob, detail=None):
    """
    Send build data generated by service to keen.io.

    Parameters:
    - buildjob : BuildJob instance
    - detail : Data storage detail level :
               'minimal', 'basic', 'full', 'extended'
    """
    if not isinstance(buildjob, BuildJob):
        raise TypeError("param buildjob should be a BuildJob instance")

    data_detail = Settings().get_value_or_setting("data_detail", detail)

    if keen_is_writable():
        logger.info(
            "Sending service build job data to Keen.io (data detail: %s)",
            data_detail
        )
        keen_add_event("build_jobs", {"job": buildjob.to_dict()})
        if data_detail in ("full", "extended"):
            keen_add_events("build_substages", buildjob.stages_to_list())


def keen_add_event(event_collection, payload):
    """
    Wrapper for keen.add_event(), adds project info.

    Param event_collection : collection event data is submitted to
    Param payload : data that is submitted
    """
    # add project info to this event
    payload = add_project_info_dict(payload)

    # submit list of events to Keen.io
    keen.add_event(event_collection, payload)


def keen_add_events(event_collection, payload):
    """
    Wrapper for keen.add_events(), adds project info to each event.

    Param event_collection : collection event data is submitted to
    Param payload : array of events that is submitted
    """
    # add project info to each event
    payload = add_project_info_list(payload)

    # submit list of events to Keen.io
    keen.add_events({event_collection: payload})


def add_project_info_dict(payload):
    """
    Add project info to a dictonary.

    Param payload: dictonary payload
    """
    # check if payload is a dictionary, throws an exception if it isn't
    check_dict(payload, "payload")

    payload_as_dict = copy.deepcopy(payload)

    payload_as_dict[KEEN_PROJECT_INFO_NAME] = Settings().get_project_info()

    if "job" in payload:
        # override project_name, set to build_job repo
        if "repo" in payload["job"]:
            payload_as_dict[KEEN_PROJECT_INFO_NAME]["project_name"] = \
                payload["job"]["repo"]

        # override timestamp, set to finished_at timestamp
        if "finished_at" in payload["job"]:
            payload_as_dict["keen"] = {
                "timestamp": payload["job"]["finished_at"]["isotimestamp"]
            }

    return payload_as_dict


def add_project_info_list(payload):
    """
    Add project info to a list of dictionaries.

    Param payload: list of dictionaries
    """
    # check if payload is a list, throws an exception if it isn't
    is_list(payload, "payload")

    payload_as_list = []

    # loop over dicts in payload and add project info to each one
    for event_dict in payload:
        payload_as_list.append(add_project_info_dict(event_dict))

    return payload_as_list


def get_dashboard_keen_config(repo):
    """
    Generate the Keen.io settings for the configuration of the dashboard.

    The dashboard is Javascript powered HTML file that contains the
    graphs generated by Keen.io.
    """
    # initialise config settings
    keen_config = {}

    if not keen_has_project_id() or not keen_has_master_key():
        logger.warning("Keen.io related config settings could not be created,"
                       " keen.project_id and/or keen.master_key"
                       " are not defined.")
        return keen_config

    # set keen project ID
    if keen.project_id is None:
        keen_project_id = os.environ["KEEN_PROJECT_ID"]
    else:
        keen_project_id = keen.project_id
    keen_config['projectId'] = str(keen_project_id)

    # generate read key
    read_key = keen_io_generate_read_key(repo)
    if read_key is not None:
        # convert bytes to string
        if isinstance(read_key, bytes):
            read_key = read_key.decode('utf-8')

        keen_config['readKey'] = str(read_key)

    # return keen config settings
    return keen_config


def get_avg_buildtime(repo=None, interval=None):

    """
    Query Keen.io database and retrieve average build time.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if repo is None or not keen_is_readable():
        return -1

    interval_data = check_time_interval(interval)

    try:
        return keen.average(
            "build_jobs",
            target_property="job.duration",
            timeframe=interval_data['timeframe'],
            max_age=interval_data['max_age'],
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_avg_buildtime() : " + str(msg))
        return -1


def get_total_build_jobs(repo=None, interval=None):

    """
    Query Keen.io database and retrieve total number of build jobs.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if repo is None or not keen_is_readable():
        return -1

    interval_data = check_time_interval(interval)

    try:
        return keen.count_unique(
            "build_jobs",
            target_property="job.job",
            timeframe=interval_data['timeframe'],
            max_age=interval_data['max_age'],
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_total_build_jobs() : " + str(msg))
        return -1


def get_passed_build_jobs(repo=None, interval=None):
    """
    Query Keen.io database and retrieve total number of build jobs that passed.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if repo is None or not keen_is_readable():
        return -1

    interval_data = check_time_interval(interval)

    try:
        return keen.count_unique(
            "build_jobs",
            target_property="job.job",
            timeframe=interval_data['timeframe'],
            max_age=interval_data['max_age'],
            filters=[
                get_repo_filter(repo),
                {
                    "property_name": "job.result",
                    "operator": "eq",
                    "property_value": "passed"
                }
            ]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_passed_build_jobs() : " + str(msg))
        return -1


def get_pct_passed_build_jobs(repo=None, interval=None):
    """
    Calculate percentage of passed build jobs.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    total_jobs = get_total_build_jobs(repo, interval)
    passed_jobs = get_passed_build_jobs(repo, interval)

    logger.debug("passed/total build jobs : %d/%d", passed_jobs, total_jobs)

    # calculate percentage if at least one job was executed
    # passed is a valid number (not -1)
    if total_jobs > 0 and passed_jobs >= 0:
        return int(float(passed_jobs) / float(total_jobs) * 100.0)

    return -1


def get_result_color(value=0, ok_thershold=90, warning_thershold=70):
    """
    Get color code that corresponds to the result.

    Parameters:
    - value : value to check
    - ok_thershold : OK threshold value
    - warning_thershold : warning thershold value
    """
    if not(type(value) in (int, float) and
           type(ok_thershold) in (int, float) and
           type(warning_thershold) in (int, float)):
        return "lightgrey"

    # check thresholds
    if value >= ok_thershold:
        return "green"
    elif value >= warning_thershold:
        return "yellow"
    else:
        return "red"


def get_total_builds(repo=None, interval=None):

    """
    Query Keen.io database and retrieve total number of builds.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if repo is None or not keen_is_readable():
        return -1

    interval_data = check_time_interval(interval)

    try:
        return keen.count_unique(
            "build_jobs",
            target_property="job.build",
            timeframe=interval_data['timeframe'],
            max_age=interval_data['max_age'],
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_total_builds() : " + str(msg))
        return -1


def get_latest_buildtime(repo=None):
    """
    Query Keen.io database and retrieve buildtime duration of last build.

    Parameters :
    - repo : repo name (fe. buildtimetrend/python-lib)
    """
    if repo is None or not keen_is_readable():
        return -1

    try:
        result = keen.extraction(
            "build_jobs",
            property_names="job.duration",
            latest=1,
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_latest_buildtime() : " + str(msg))
        return -1

    if is_list(result) and len(result) > 0 and \
            check_dict(result[0], None, ['job']) and \
            check_dict(result[0]['job'], None, ['duration']):
        return result[0]['job']['duration']

    return -1


def has_build_id(repo=None, build_id=None):
    """
    Check if build_id exists in Keen.io database.

    Parameters :
    - repo : repo name (fe. buildtimetrend/python-lib)
    - build_id : ID of the build
    """
    if repo is None or build_id is None:
        logger.error("Repo or build_id is not set")
        raise ValueError("Repo or build_id is not set")
    if not keen_is_readable():
        raise SystemError("Keen.io Project ID or API Read Key is not set")

    try:
        count = keen.count(
            "build_jobs",
            filters=[get_repo_filter(repo), {
                "property_name": "job.build",
                "operator": "eq",
                "property_value": str(build_id)}]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        raise SystemError("Connection to Keen.io API failed")
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.has_build_id : " + str(msg))
        raise SystemError(msg)

    return count > 0


def get_all_projects():
    """Query Keen.io database and retrieve a list of all projects."""
    if not keen_is_readable():
        return []

    try:
        result = keen.select_unique(
            "build_jobs",
            "buildtime_trend.project_name",
            max_age=3600 * 24  # cache for 24 hours
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return []
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_all_projects() : " + str(msg))
        return []

    if type(result) is list:
        return result

    return []


def get_repo_filter(repo=None):
    """
    Return filter for analysis request.

    Parameters
    - repo : repo slug name, fe. buildtimetrend/python-lib
    """
    if repo is None:
        return None

    return {
        "property_name": "{}.project_name".format(KEEN_PROJECT_INFO_NAME),
        "operator": "eq",
        "property_value": str(repo)
    }


def check_time_interval(interval=None):
    """
    Check time interval and returns corresponding parameters.

    Parameters :
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if is_string(interval):
        # convert to lowercase
        interval = interval.lower()

        if interval in TIME_INTERVALS:
            return TIME_INTERVALS[interval]

    return TIME_INTERVALS['week']


1			# vim: set expandtab sw=4 ts=4:
2			"""
3			Interface to Keen IO.
4
5			Copyright (C) 2014-2016 Dieter Adriaenssens <[email protected]>
6
7			This file is part of buildtimetrend/python-lib
8			<https://github.com/buildtimetrend/python-lib/>
9
10			This program is free software: you can redistribute it and/or modify
11			it under the terms of the GNU Affero General Public License as published by
12			the Free Software Foundation, either version 3 of the License, or
13			any later version.
14
15			This program is distributed in the hope that it will be useful,
16			but WITHOUT ANY WARRANTY; without even the implied warranty of
17			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18			GNU Affero General Public License for more details.
19
20			You should have received a copy of the GNU Affero General Public License
21			along with this program. If not, see <http://www.gnu.org/licenses/>.
22			"""
23
24			from __future__ import division
25			from builtins import str
26			import os
27			from buildtimetrend import logger
28			import copy
29			import keen
30			import requests
31			from keen import scoped_keys
32			from buildtimetrend.settings import Settings
33			from buildtimetrend.tools import check_dict
34			from buildtimetrend.tools import is_list
35			from buildtimetrend.tools import is_string
36			from buildtimetrend.buildjob import BuildJob
37
38
39			TIME_INTERVALS = {
40			'week': {'name': 'week', 'timeframe': 'this_7_days', 'max_age': 600},
41			'month': {'name': 'month', 'timeframe': 'this_30_days', 'max_age': 600},
42			'year': {'name': 'year', 'timeframe': 'this_52_weeks', 'max_age': 1800}
43			}
44			KEEN_PROJECT_INFO_NAME = "buildtime_trend"
45
46
47			def keen_has_project_id():
48			"""Check if Keen.io project ID is set."""
49			if "KEEN_PROJECT_ID" in os.environ or keen.project_id is not None:
50			return True
51
52			logger.warning("Keen.io Project ID is not set")
53			return False
54
55
56			def keen_has_master_key():
57			"""Check if Keen.io Master API key is set."""
58			if "KEEN_MASTER_KEY" in os.environ or keen.master_key is not None:
59			return True
60
61			logger.warning("Keen.io Master API Key is not set")
62			return False
63
64
65			def keen_has_write_key():
66			"""Check if Keen.io Write Key is set."""
67			if "KEEN_WRITE_KEY" in os.environ or keen.write_key is not None:
68			return True
69
70			logger.warning("Keen.io Write Key is not set")
71			return False
72
73
74			def keen_has_read_key():
75			"""Check if Keen.io Read key is set."""
76			if "KEEN_READ_KEY" in os.environ or keen.read_key is not None:
77			return True
78
79			logger.warning("Keen.io Read Key is not set")
80			return False
81
82
83			def keen_is_writable():
84			"""Check if login keys for Keen IO API are set, to allow writing."""
85			if keen_has_project_id() and keen_has_write_key():
86			return True
87
88			logger.warning("Keen.io Write Key is not set")
89			return False
90
91
92			def keen_is_readable():
93			"""Check if login keys for Keen IO API are set, to allow reading."""
94			if keen_has_project_id() and keen_has_read_key():
95			return True
96
97			logger.warning("Keen.io Read Key is not set")
98			return False
99
100
101			def keen_io_generate_read_key(repo):
102			"""
103			Create scoped key for reading only the build-stages related data.
104
105			Param repo : github repository slug (fe. buildtimetrend/python-lib)
106			"""
107			if not keen_has_master_key():
108			logger.warning("Keen.io Read Key was not created,"
109			" keen.master_key is not defined.")
110			return None
111
112			master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")
113
114			privileges = {
115			"allowed_operations": ["read"]
116			}
117
118			if repo is not None:
119			privileges["filters"] = [get_repo_filter(repo)]
120
121			logger.info("Keen.io Read Key is created for %s", repo)
122			return scoped_keys.encrypt(master_key, privileges)
123
124
125			def keen_io_generate_write_key():
126			"""Create scoped key for write access to Keen.io database."""
127			if not keen_has_master_key():
128			logger.warning("Keen.io Write Key was not created,"
129			" keen.master_key is not defined.")
130			return None
131
132			master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")
133
134			privileges = {
135			"allowed_operations": ["write"]
136			}
137
138			logger.info("Keen.io Write Key is created")
139			return scoped_keys.encrypt(master_key, privileges)
140
141
142			def send_build_data(buildjob, detail=None):
143			"""
144			Send build data generated by client to keen.io.
145
146			Parameters:
147			- buildjob : BuildJob instance
148			- detail : Data storage detail level :
149			'minimal', 'basic', 'full', 'extended'
150			"""
151			if not isinstance(buildjob, BuildJob):
152			raise TypeError("param buildjob should be a BuildJob instance")
153
154			data_detail = Settings().get_value_or_setting("data_detail", detail)
155
156			if keen_is_writable():
157			logger.info(
158			"Sending client build job data to Keen.io (data detail: %s)",
159			data_detail
160			)
161			keen_add_event("build_jobs", {"job": buildjob.to_dict()})
162			if data_detail in ("full", "extended"):
163			keen_add_events("build_stages", buildjob.stages_to_list())
164
165
166			def send_build_data_service(buildjob, detail=None):
167			"""
168			Send build data generated by service to keen.io.
169
170			Parameters:
171			- buildjob : BuildJob instance
172			- detail : Data storage detail level :
173			'minimal', 'basic', 'full', 'extended'
174			"""
175			if not isinstance(buildjob, BuildJob):
176			raise TypeError("param buildjob should be a BuildJob instance")
177
178			data_detail = Settings().get_value_or_setting("data_detail", detail)
179
180			if keen_is_writable():
181			logger.info(
182			"Sending service build job data to Keen.io (data detail: %s)",
183			data_detail
184			)
185			keen_add_event("build_jobs", {"job": buildjob.to_dict()})
186			if data_detail in ("full", "extended"):
187			keen_add_events("build_substages", buildjob.stages_to_list())
188
189
190			def keen_add_event(event_collection, payload):
191			"""
192			Wrapper for keen.add_event(), adds project info.
193
194			Param event_collection : collection event data is submitted to
195			Param payload : data that is submitted
196			"""
197			# add project info to this event
198			payload = add_project_info_dict(payload)
199
200			# submit list of events to Keen.io
201			keen.add_event(event_collection, payload)
202
203
204			def keen_add_events(event_collection, payload):
205			"""
206			Wrapper for keen.add_events(), adds project info to each event.
207
208			Param event_collection : collection event data is submitted to
209			Param payload : array of events that is submitted
210			"""
211			# add project info to each event
212			payload = add_project_info_list(payload)
213
214			# submit list of events to Keen.io
215			keen.add_events({event_collection: payload})
216
217
218			def add_project_info_dict(payload):
219			"""
220			Add project info to a dictonary.
221
222			Param payload: dictonary payload
223			"""
224			# check if payload is a dictionary, throws an exception if it isn't
225			check_dict(payload, "payload")
226
227			payload_as_dict = copy.deepcopy(payload)
228
229			payload_as_dict[KEEN_PROJECT_INFO_NAME] = Settings().get_project_info()
230
231			if "job" in payload:
232			# override project_name, set to build_job repo
233			if "repo" in payload["job"]:
234			payload_as_dict[KEEN_PROJECT_INFO_NAME]["project_name"] = \
235			payload["job"]["repo"]
236
237			# override timestamp, set to finished_at timestamp
238			if "finished_at" in payload["job"]:
239			payload_as_dict["keen"] = {
240			"timestamp": payload["job"]["finished_at"]["isotimestamp"]
241			}
242
243			return payload_as_dict
244
245
246			def add_project_info_list(payload):
247			"""
248			Add project info to a list of dictionaries.
249
250			Param payload: list of dictionaries
251			"""
252			# check if payload is a list, throws an exception if it isn't
253			is_list(payload, "payload")
254
255			payload_as_list = []
256
257			# loop over dicts in payload and add project info to each one
258			for event_dict in payload:
259			payload_as_list.append(add_project_info_dict(event_dict))
260
261			return payload_as_list
262
263
264			def get_dashboard_keen_config(repo):
265			"""
266			Generate the Keen.io settings for the configuration of the dashboard.
267
268			The dashboard is Javascript powered HTML file that contains the
269			graphs generated by Keen.io.
270			"""
271			# initialise config settings
272			keen_config = {}
273
274			if not keen_has_project_id() or not keen_has_master_key():
275			logger.warning("Keen.io related config settings could not be created,"
276			" keen.project_id and/or keen.master_key"
277			" are not defined.")
278			return keen_config
279
280			# set keen project ID
281			if keen.project_id is None:
282			keen_project_id = os.environ["KEEN_PROJECT_ID"]
283			else:
284			keen_project_id = keen.project_id
285			keen_config['projectId'] = str(keen_project_id)
286
287			# generate read key
288			read_key = keen_io_generate_read_key(repo)
289			if read_key is not None:
290			# convert bytes to string
291			if isinstance(read_key, bytes):
292			read_key = read_key.decode('utf-8')
293
294			keen_config['readKey'] = str(read_key)
295
296			# return keen config settings
297			return keen_config
298
299
300			def get_avg_buildtime(repo=None, interval=None):
			0 ignored issues – show Duplication introduced 2015-11-22 12:44 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
301			"""
302			Query Keen.io database and retrieve average build time.
303
304			Parameters :
305			- repo : repo name (fe. buildtimetrend/service)
306			- interval : timeframe, possible values : 'week', 'month', 'year',
307			anything else defaults to 'week'
308			"""
309			if repo is None or not keen_is_readable():
310			return -1
311
312			interval_data = check_time_interval(interval)
313
314			try:
315			return keen.average(
316			"build_jobs",
317			target_property="job.duration",
318			timeframe=interval_data['timeframe'],
319			max_age=interval_data['max_age'],
320			filters=[get_repo_filter(repo)]
321			)
322			except requests.ConnectionError:
323			logger.error("Connection to Keen.io API failed")
324			return -1
325			except keen.exceptions.KeenApiError as msg:
326			logger.error("Error in keenio.get_avg_buildtime() : " + str(msg))
327			return -1
328
329
330			def get_total_build_jobs(repo=None, interval=None):
			0 ignored issues – show Duplication introduced 2015-11-22 12:44 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
331			"""
332			Query Keen.io database and retrieve total number of build jobs.
333
334			Parameters :
335			- repo : repo name (fe. buildtimetrend/service)
336			- interval : timeframe, possible values : 'week', 'month', 'year',
337			anything else defaults to 'week'
338			"""
339			if repo is None or not keen_is_readable():
340			return -1
341
342			interval_data = check_time_interval(interval)
343
344			try:
345			return keen.count_unique(
346			"build_jobs",
347			target_property="job.job",
348			timeframe=interval_data['timeframe'],
349			max_age=interval_data['max_age'],
350			filters=[get_repo_filter(repo)]
351			)
352			except requests.ConnectionError:
353			logger.error("Connection to Keen.io API failed")
354			return -1
355			except keen.exceptions.KeenApiError as msg:
356			logger.error("Error in keenio.get_total_build_jobs() : " + str(msg))
357			return -1
358
359
360			def get_passed_build_jobs(repo=None, interval=None):
361			"""
362			Query Keen.io database and retrieve total number of build jobs that passed.
363
364			Parameters :
365			- repo : repo name (fe. buildtimetrend/service)
366			- interval : timeframe, possible values : 'week', 'month', 'year',
367			anything else defaults to 'week'
368			"""
369			if repo is None or not keen_is_readable():
370			return -1
371
372			interval_data = check_time_interval(interval)
373
374			try:
375			return keen.count_unique(
376			"build_jobs",
377			target_property="job.job",
378			timeframe=interval_data['timeframe'],
379			max_age=interval_data['max_age'],
380			filters=[
381			get_repo_filter(repo),
382			{
383			"property_name": "job.result",
384			"operator": "eq",
385			"property_value": "passed"
386			}
387			]
388			)
389			except requests.ConnectionError:
390			logger.error("Connection to Keen.io API failed")
391			return -1
392			except keen.exceptions.KeenApiError as msg:
393			logger.error("Error in keenio.get_passed_build_jobs() : " + str(msg))
394			return -1
395
396
397			def get_pct_passed_build_jobs(repo=None, interval=None):
398			"""
399			Calculate percentage of passed build jobs.
400
401			Parameters :
402			- repo : repo name (fe. buildtimetrend/service)
403			- interval : timeframe, possible values : 'week', 'month', 'year',
404			anything else defaults to 'week'
405			"""
406			total_jobs = get_total_build_jobs(repo, interval)
407			passed_jobs = get_passed_build_jobs(repo, interval)
408
409			logger.debug("passed/total build jobs : %d/%d", passed_jobs, total_jobs)
410
411			# calculate percentage if at least one job was executed
412			# passed is a valid number (not -1)
413			if total_jobs > 0 and passed_jobs >= 0:
414			return int(float(passed_jobs) / float(total_jobs) * 100.0)
415
416			return -1
417
418
419			def get_result_color(value=0, ok_thershold=90, warning_thershold=70):
420			"""
421			Get color code that corresponds to the result.
422
423			Parameters:
424			- value : value to check
425			- ok_thershold : OK threshold value
426			- warning_thershold : warning thershold value
427			"""
428			if not(type(value) in (int, float) and
429			type(ok_thershold) in (int, float) and
430			type(warning_thershold) in (int, float)):
431			return "lightgrey"
432
433			# check thresholds
434			if value >= ok_thershold:
435			return "green"
436			elif value >= warning_thershold:
437			return "yellow"
438			else:
439			return "red"
440
441
442			def get_total_builds(repo=None, interval=None):
			0 ignored issues – show Duplication introduced 2015-11-22 12:44 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
443			"""
444			Query Keen.io database and retrieve total number of builds.
445
446			Parameters :
447			- repo : repo name (fe. buildtimetrend/service)
448			- interval : timeframe, possible values : 'week', 'month', 'year',
449			anything else defaults to 'week'
450			"""
451			if repo is None or not keen_is_readable():
452			return -1
453
454			interval_data = check_time_interval(interval)
455
456			try:
457			return keen.count_unique(
458			"build_jobs",
459			target_property="job.build",
460			timeframe=interval_data['timeframe'],
461			max_age=interval_data['max_age'],
462			filters=[get_repo_filter(repo)]
463			)
464			except requests.ConnectionError:
465			logger.error("Connection to Keen.io API failed")
466			return -1
467			except keen.exceptions.KeenApiError as msg:
468			logger.error("Error in keenio.get_total_builds() : " + str(msg))
469			return -1
470
471
472			def get_latest_buildtime(repo=None):
473			"""
474			Query Keen.io database and retrieve buildtime duration of last build.
475
476			Parameters :
477			- repo : repo name (fe. buildtimetrend/python-lib)
478			"""
479			if repo is None or not keen_is_readable():
480			return -1
481
482			try:
483			result = keen.extraction(
484			"build_jobs",
485			property_names="job.duration",
486			latest=1,
487			filters=[get_repo_filter(repo)]
488			)
489			except requests.ConnectionError:
490			logger.error("Connection to Keen.io API failed")
491			return -1
492			except keen.exceptions.KeenApiError as msg:
493			logger.error("Error in keenio.get_latest_buildtime() : " + str(msg))
494			return -1
495
496			if is_list(result) and len(result) > 0 and \
497			check_dict(result[0], None, ['job']) and \
498			check_dict(result[0]['job'], None, ['duration']):
499			return result[0]['job']['duration']
500
501			return -1
502
503
504			def has_build_id(repo=None, build_id=None):
505			"""
506			Check if build_id exists in Keen.io database.
507
508			Parameters :
509			- repo : repo name (fe. buildtimetrend/python-lib)
510			- build_id : ID of the build
511			"""
512			if repo is None or build_id is None:
513			logger.error("Repo or build_id is not set")
514			raise ValueError("Repo or build_id is not set")
515			if not keen_is_readable():
516			raise SystemError("Keen.io Project ID or API Read Key is not set")
517
518			try:
519			count = keen.count(
520			"build_jobs",
521			filters=[get_repo_filter(repo), {
522			"property_name": "job.build",
523			"operator": "eq",
524			"property_value": str(build_id)}]
525			)
526			except requests.ConnectionError:
527			logger.error("Connection to Keen.io API failed")
528			raise SystemError("Connection to Keen.io API failed")
529			except keen.exceptions.KeenApiError as msg:
530			logger.error("Error in keenio.has_build_id : " + str(msg))
531			raise SystemError(msg)
532
533			return count > 0
534
535
536			def get_all_projects():
537			"""Query Keen.io database and retrieve a list of all projects."""
538			if not keen_is_readable():
539			return []
540
541			try:
542			result = keen.select_unique(
543			"build_jobs",
544			"buildtime_trend.project_name",
545			max_age=3600 * 24 # cache for 24 hours
546			)
547			except requests.ConnectionError:
548			logger.error("Connection to Keen.io API failed")
549			return []
550			except keen.exceptions.KeenApiError as msg:
551			logger.error("Error in keenio.get_all_projects() : " + str(msg))
552			return []
553
554			if type(result) is list:
555			return result
556
557			return []
558
559
560			def get_repo_filter(repo=None):
561			"""
562			Return filter for analysis request.
563
564			Parameters
565			- repo : repo slug name, fe. buildtimetrend/python-lib
566			"""
567			if repo is None:
568			return None
569
570			return {
571			"property_name": "{}.project_name".format(KEEN_PROJECT_INFO_NAME),
572			"operator": "eq",
573			"property_value": str(repo)
574			}
575
576
577			def check_time_interval(interval=None):
578			"""
579			Check time interval and returns corresponding parameters.
580
581			Parameters :
582			- interval : timeframe, possible values : 'week', 'month', 'year',
583			anything else defaults to 'week'
584			"""
585			if is_string(interval):
586			# convert to lowercase
587			interval = interval.lower()
588
589			if interval in TIME_INTERVALS:
590			return TIME_INTERVALS[interval]
591
592			return TIME_INTERVALS['week']
593

buildtimetrend / python-lib

Push — master ( fec650...1a3e44 )

buildtimetrend.send_build_data_service() B

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like