buildtimetrend.get_avg_buildtime() - Code Metrics - Inspection of "add missing ," - buildtimetrend/python-lib - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 44a554...4ba01b )

by Dieter

created 2015-11-22 12:43 UTC

buildtimetrend.get_avg_buildtime() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	5
dl	0
loc	28
rs	8.0896

# vim: set expandtab sw=4 ts=4:
"""
Interface to Keen IO.

Copyright (C) 2014-2015 Dieter Adriaenssens <[email protected]>

This file is part of buildtimetrend/python-lib
<https://github.com/buildtimetrend/python-lib/>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""

from __future__ import division
import os
from buildtimetrend import logger
import copy
import keen
import requests
from keen import scoped_keys
from buildtimetrend.settings import Settings
from buildtimetrend.tools import check_file
from buildtimetrend.tools import check_dict
from buildtimetrend.tools import is_list
from buildtimetrend.tools import is_string


TIME_INTERVALS = {
    'week': {'name': 'week', 'timeframe': 'this_7_days', 'max_age': 600},
    'month': {'name': 'month', 'timeframe': 'this_30_days', 'max_age': 600},
    'year': {'name': 'year', 'timeframe': 'this_52_weeks', 'max_age': 1800}
}
KEEN_PROJECT_INFO_NAME = "buildtime_trend"


def keen_has_project_id():
    """Check if Keen.io project ID is set."""
    if "KEEN_PROJECT_ID" in os.environ or keen.project_id is not None:
        return True

    logger.warning("Keen.io Project ID is not set")
    return False


def keen_has_master_key():
    """Check if Keen.io Master API key is set."""
    if "KEEN_MASTER_KEY" in os.environ or keen.master_key is not None:
        return True

    logger.warning("Keen.io Master API Key is not set")
    return False


def keen_has_write_key():
    """Check if Keen.io Write Key is set."""
    if "KEEN_WRITE_KEY" in os.environ or keen.write_key is not None:
        return True

    logger.warning("Keen.io Write Key is not set")
    return False


def keen_has_read_key():
    """Check if Keen.io Read key is set."""
    if "KEEN_READ_KEY" in os.environ or keen.read_key is not None:
        return True

    logger.warning("Keen.io Read Key is not set")
    return False


def keen_is_writable():
    """Check if login keys for Keen IO API are set, to allow writing."""
    if keen_has_project_id() and keen_has_write_key():
        return True

    logger.warning("Keen.io Write Key is not set")
    return False


def keen_is_readable():
    """Check if login keys for Keen IO API are set, to allow reading."""
    if keen_has_project_id() and keen_has_read_key():
        return True

    logger.warning("Keen.io Read Key is not set")
    return False


def keen_io_generate_read_key(repo):
    """
    Create scoped key for reading only the build-stages related data.

    Param repo : github repository slug (fe. buildtimetrend/python-lib)
    """
    if not keen_has_master_key():
        logger.warning("Keen.io Read Key was not created,"
                       " keen.master_key is not defined.")
        return None

    master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")

    privileges = {
        "allowed_operations": ["read"]
    }

    if repo is not None:
        privileges["filters"] = [get_repo_filter(repo)]

    logger.info("Keen.io Read Key is created for %s", repo)
    return scoped_keys.encrypt(master_key, privileges)


def keen_io_generate_write_key():
    """Create scoped key for write access to Keen.io database."""
    if not keen_has_master_key():
        logger.warning("Keen.io Write Key was not created,"
                       " keen.master_key is not defined.")
        return None

    master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")

    privileges = {
        "allowed_operations": ["write"]
    }

    logger.info("Keen.io Write Key is created")
    return scoped_keys.encrypt(master_key, privileges)


def send_build_data(build):
    """Send build data generated by client to keen.io."""
    if keen_is_writable():
        logger.info("Sending client build data to Keen.io")
        keen_add_event("build_jobs", {"job": build.to_dict()})
        keen_add_events("build_stages", build.stages_to_list())


def send_build_data_service(build):
    """Send build data generated by service to keen.io."""
    if keen_is_writable():
        logger.info("Sending service build data to Keen.io")
        keen_add_event("build_jobs", {"job": build.to_dict()})
        keen_add_events("build_substages", build.stages_to_list())


def keen_add_event(event_collection, payload):
    """
    Wrapper for keen.add_event(), adds project info.

    Param event_collection : collection event data is submitted to
    Param payload : data that is submitted
    """
    # add project info to this event
    payload = add_project_info_dict(payload)

    # submit list of events to Keen.io
    keen.add_event(event_collection, payload)


def keen_add_events(event_collection, payload):
    """
    Wrapper for keen.add_events(), adds project info to each event.

    Param event_collection : collection event data is submitted to
    Param payload : array of events that is submitted
    """
    # add project info to each event
    payload = add_project_info_list(payload)

    # submit list of events to Keen.io
    keen.add_events({event_collection: payload})


def add_project_info_dict(payload):
    """
    Add project info to a dictonary.

    Param payload: dictonary payload
    """
    # check if payload is a dictionary, throws an exception if it isn't
    check_dict(payload, "payload")

    payload_as_dict = copy.deepcopy(payload)

    payload_as_dict[KEEN_PROJECT_INFO_NAME] = Settings().get_project_info()

    if "job" in payload:
        # override project_name, set to build_job repo
        if "repo" in payload["job"]:
            payload_as_dict[KEEN_PROJECT_INFO_NAME]["project_name"] = \
                payload["job"]["repo"]

        # override timestamp, set to finished_at timestamp
        if "finished_at" in payload["job"]:
            payload_as_dict["keen"] = {
                "timestamp": payload["job"]["finished_at"]["isotimestamp"]
            }

    return payload_as_dict


def add_project_info_list(payload):
    """
    Add project info to a list of dictionaries.

    Param payload: list of dictionaries
    """
    # check if payload is a list, throws an exception if it isn't
    is_list(payload, "payload")

    payload_as_list = []

    # loop over dicts in payload and add project info to each one
    for event_dict in payload:
        payload_as_list.append(add_project_info_dict(event_dict))

    return payload_as_list


def get_dashboard_keen_config(repo):
    """
    Generate the Keen.io settings for the configuration of the dashboard.

    The dashboard is Javascript powered HTML file that contains the
    graphs generated by Keen.io.
    """
    # initialise config settings
    keen_config = {}

    if not keen_has_project_id() or not keen_has_master_key():
        logger.warning("Keen.io related config settings could not be created,"
                       " keen.project_id and/or keen.master_key"
                       " are not defined.")
        return keen_config

    # set keen project ID
    if keen.project_id is None:
        keen_project_id = os.environ["KEEN_PROJECT_ID"]
    else:
        keen_project_id = keen.project_id
    keen_config['projectId'] = str(keen_project_id)

    # generate read key
    read_key = keen_io_generate_read_key(repo)
    if read_key is not None:
        keen_config['readKey'] = str(read_key)

    # return keen config settings
    return keen_config


def get_dashboard_config(repo, extra=None):
    """
    Generate the configuration settings for the dashboard.

    The dashboard is Javascript powered HTML file that contains the
    graphs generated by Keen.io.

    Parameters:
    - repo : repo name (fe. buildtimetrend/service)
    - extra : dictionary of extra config settings, format : {"name" : "value"}
    """
    # initialise config settings dictionaries
    config = {}
    keen_config = get_dashboard_keen_config(repo)

    # add repo and project name
    if repo is not None and not repo == "":
        # merge extra settings into existing config dictionary
        config.update({
            'projectName': str(repo),
            'repoName': str(repo)
        })

    # add extra config parameters
    if extra is not None and check_dict(extra, "extra"):
        config.update(extra)

    # create configuration as a string
    return "var config = %s;\nvar keenConfig = %s;" % (config, keen_config)


def generate_dashboard_config_file(repo):
    """
    Generate the configuration file for the dashboard.

    The dashboard is Javascript powered HTML file that contains the
    graphs generated by Keen.io.

    Parameters:
    - repo : repo name (fe. buildtimetrend/service)
    """
    # get config settings
    config_string = get_dashboard_config(repo)

    # write config file
    config_file = Settings().get_setting("dashboard_configfile")
    with open(config_file, 'w') as outfile:
        outfile.write(config_string)

    if check_file(config_file):
        logger.info("Created trends dashboard config file %s", config_file)
        return True
    else:
        logger.warning("The dashboard config file was not created")
        return False


def get_avg_buildtime(repo=None, interval=None):

    """
    Query Keen.io database and retrieve average build time.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if repo is None or not keen_is_readable():
        return -1

    interval_data = check_time_interval(interval)

    try:
        return keen.average(
            "build_jobs",
            target_property="job.duration",
            timeframe=interval_data['timeframe'],
            max_age=interval_data['max_age'],
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_avg_buildtime() : " + str(msg))
        return -1


def get_total_build_jobs(repo=None, interval=None):

    """
    Query Keen.io database and retrieve total number of build jobs.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if repo is None or not keen_is_readable():
        return -1

    interval_data = check_time_interval(interval)

    try:
        return keen.count_unique(
            "build_jobs",
            target_property="job.job",
            timeframe=interval_data['timeframe'],
            max_age=interval_data['max_age'],
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_total_build_jobs() : " + str(msg))
        return -1


def get_passed_build_jobs(repo=None, interval=None):
    """
    Query Keen.io database and retrieve total number of build jobs that passed.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if repo is None or not keen_is_readable():
        return -1

    interval_data = check_time_interval(interval)

    try:
        return keen.count_unique(
            "build_jobs",
            target_property="job.job",
            timeframe=interval_data['timeframe'],
            max_age=interval_data['max_age'],
            filters=[
                get_repo_filter(repo),
                {
                    "property_name": "job.result",
                    "operator": "eq",
                    "property_value": "passed"
                }
            ]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_passed_build_jobs() : " + str(msg))
        return -1


def get_pct_passed_build_jobs(repo=None, interval=None):
    """
    Calculate percentage of passed build jobs.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    total_jobs = get_total_build_jobs(repo, interval)
    passed_jobs = get_passed_build_jobs(repo, interval)

    logger.debug("passed/total build jobs : %d/%d", passed_jobs, total_jobs)

    # calculate percentage if at least one job was executed
    # passed is a valid number (not -1)
    if total_jobs > 0 and passed_jobs >= 0:
        return int(float(passed_jobs) / float(total_jobs) * 100.0)

    return -1


def get_result_color(value=0, ok_thershold=90, warning_thershold=70):
    """
    Get color code that corresponds to the result.

    Parameters:
    - value : value to check
    - ok_thershold : OK threshold value
    - warning_thershold : warning thershold value
    """
    if not(type(value) in (int, float) and
           type(ok_thershold) in (int, float) and
           type(warning_thershold) in (int, float)):
        return "lightgrey"

    # check thresholds
    if value >= ok_thershold:
        return "green"
    elif value >= warning_thershold:
        return "yellow"
    else:
        return "red"


def get_total_builds(repo=None, interval=None):

    """
    Query Keen.io database and retrieve total number of builds.

    Parameters :
    - repo : repo name (fe. buildtimetrend/service)
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if repo is None or not keen_is_readable():
        return -1

    interval_data = check_time_interval(interval)

    try:
        return keen.count_unique(
            "build_jobs",
            target_property="job.build",
            timeframe=interval_data['timeframe'],
            max_age=interval_data['max_age'],
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_total_builds() : " + str(msg))
        return -1


def get_latest_buildtime(repo=None):
    """
    Query Keen.io database and retrieve buildtime duration of last build.

    Parameters :
    - repo : repo name (fe. buildtimetrend/python-lib)
    """
    if repo is None or not keen_is_readable():
        return -1

    try:
        result = keen.extraction(
            "build_jobs",
            property_names="job.duration",
            latest=1,
            filters=[get_repo_filter(repo)]
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return -1
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_latest_buildtime() : " + str(msg))
        return -1

    if result is not None and len(result) > 0:
        return result[0]['job']['duration']

    return -1


def has_build_id(repo=None, build_id=None):
    """
    Check if build_id exists in Keen.io database.

    Parameters :
    - repo : repo name (fe. buildtimetrend/python-lib)
    - build_id : ID of the build
    """
    if repo is None or build_id is None:
        logger.error("Repo or build_id is not set")
        raise ValueError("Repo or build_id is not set")
    if not keen_is_readable():
        raise SystemError("Keen.io Project ID or API Read Key is not set")

    try:
        count = keen.count(
            "build_jobs",
            filters=[get_repo_filter(repo), {
                "property_name": "job.build",
                "operator": "eq",
                "property_value": str(build_id)}]
        )
        return count > 0
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        raise SystemError("Connection to Keen.io API failed")
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.has_build_id : " + str(msg))
        raise SystemError(msg)


def get_all_projects():
    """Query Keen.io database and retrieve a list of all projects."""
    if not keen_is_readable():
        return []

    try:
        result = keen.select_unique(
            "build_jobs",
            "buildtime_trend.project_name",
            max_age=3600 * 24  # cache for 24 hours
        )
    except requests.ConnectionError:
        logger.error("Connection to Keen.io API failed")
        return []
    except keen.exceptions.KeenApiError as msg:
        logger.error("Error in keenio.get_all_projects() : " + str(msg))
        return -1

    if type(result) is list:
        return result

    return []


def get_repo_filter(repo=None):
    """
    Return filter for analysis request.

    Parameters
    - repo : repo slug name, fe. buildtimetrend/python-lib
    """
    if repo is None:
        return None

    return {
        "property_name": "%s.project_name" % KEEN_PROJECT_INFO_NAME,
        "operator": "eq",
        "property_value": str(repo)
    }


def check_time_interval(interval=None):
    """
    Check time interval and returns corresponding parameters.

    Parameters :
    - interval : timeframe, possible values : 'week', 'month', 'year',
                 anything else defaults to 'week'
    """
    if is_string(interval):
        # convert to lowercase
        interval = interval.lower()

        if interval in TIME_INTERVALS:
            return TIME_INTERVALS[interval]

    return TIME_INTERVALS['week']


1			# vim: set expandtab sw=4 ts=4:
2			"""
3			Interface to Keen IO.
4
5			Copyright (C) 2014-2015 Dieter Adriaenssens <[email protected]>
6
7			This file is part of buildtimetrend/python-lib
8			<https://github.com/buildtimetrend/python-lib/>
9
10			This program is free software: you can redistribute it and/or modify
11			it under the terms of the GNU Affero General Public License as published by
12			the Free Software Foundation, either version 3 of the License, or
13			any later version.
14
15			This program is distributed in the hope that it will be useful,
16			but WITHOUT ANY WARRANTY; without even the implied warranty of
17			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18			GNU Affero General Public License for more details.
19
20			You should have received a copy of the GNU Affero General Public License
21			along with this program. If not, see <http://www.gnu.org/licenses/>.
22			"""
23
24			from __future__ import division
25			import os
26			from buildtimetrend import logger
27			import copy
28			import keen
29			import requests
30			from keen import scoped_keys
31			from buildtimetrend.settings import Settings
32			from buildtimetrend.tools import check_file
33			from buildtimetrend.tools import check_dict
34			from buildtimetrend.tools import is_list
35			from buildtimetrend.tools import is_string
36
37
38			TIME_INTERVALS = {
39			'week': {'name': 'week', 'timeframe': 'this_7_days', 'max_age': 600},
40			'month': {'name': 'month', 'timeframe': 'this_30_days', 'max_age': 600},
41			'year': {'name': 'year', 'timeframe': 'this_52_weeks', 'max_age': 1800}
42			}
43			KEEN_PROJECT_INFO_NAME = "buildtime_trend"
44
45
46			def keen_has_project_id():
47			"""Check if Keen.io project ID is set."""
48			if "KEEN_PROJECT_ID" in os.environ or keen.project_id is not None:
49			return True
50
51			logger.warning("Keen.io Project ID is not set")
52			return False
53
54
55			def keen_has_master_key():
56			"""Check if Keen.io Master API key is set."""
57			if "KEEN_MASTER_KEY" in os.environ or keen.master_key is not None:
58			return True
59
60			logger.warning("Keen.io Master API Key is not set")
61			return False
62
63
64			def keen_has_write_key():
65			"""Check if Keen.io Write Key is set."""
66			if "KEEN_WRITE_KEY" in os.environ or keen.write_key is not None:
67			return True
68
69			logger.warning("Keen.io Write Key is not set")
70			return False
71
72
73			def keen_has_read_key():
74			"""Check if Keen.io Read key is set."""
75			if "KEEN_READ_KEY" in os.environ or keen.read_key is not None:
76			return True
77
78			logger.warning("Keen.io Read Key is not set")
79			return False
80
81
82			def keen_is_writable():
83			"""Check if login keys for Keen IO API are set, to allow writing."""
84			if keen_has_project_id() and keen_has_write_key():
85			return True
86
87			logger.warning("Keen.io Write Key is not set")
88			return False
89
90
91			def keen_is_readable():
92			"""Check if login keys for Keen IO API are set, to allow reading."""
93			if keen_has_project_id() and keen_has_read_key():
94			return True
95
96			logger.warning("Keen.io Read Key is not set")
97			return False
98
99
100			def keen_io_generate_read_key(repo):
101			"""
102			Create scoped key for reading only the build-stages related data.
103
104			Param repo : github repository slug (fe. buildtimetrend/python-lib)
105			"""
106			if not keen_has_master_key():
107			logger.warning("Keen.io Read Key was not created,"
108			" keen.master_key is not defined.")
109			return None
110
111			master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")
112
113			privileges = {
114			"allowed_operations": ["read"]
115			}
116
117			if repo is not None:
118			privileges["filters"] = [get_repo_filter(repo)]
119
120			logger.info("Keen.io Read Key is created for %s", repo)
121			return scoped_keys.encrypt(master_key, privileges)
122
123
124			def keen_io_generate_write_key():
125			"""Create scoped key for write access to Keen.io database."""
126			if not keen_has_master_key():
127			logger.warning("Keen.io Write Key was not created,"
128			" keen.master_key is not defined.")
129			return None
130
131			master_key = keen.master_key or os.environ.get("KEEN_MASTER_KEY")
132
133			privileges = {
134			"allowed_operations": ["write"]
135			}
136
137			logger.info("Keen.io Write Key is created")
138			return scoped_keys.encrypt(master_key, privileges)
139
140
141			def send_build_data(build):
142			"""Send build data generated by client to keen.io."""
143			if keen_is_writable():
144			logger.info("Sending client build data to Keen.io")
145			keen_add_event("build_jobs", {"job": build.to_dict()})
146			keen_add_events("build_stages", build.stages_to_list())
147
148
149			def send_build_data_service(build):
150			"""Send build data generated by service to keen.io."""
151			if keen_is_writable():
152			logger.info("Sending service build data to Keen.io")
153			keen_add_event("build_jobs", {"job": build.to_dict()})
154			keen_add_events("build_substages", build.stages_to_list())
155
156
157			def keen_add_event(event_collection, payload):
158			"""
159			Wrapper for keen.add_event(), adds project info.
160
161			Param event_collection : collection event data is submitted to
162			Param payload : data that is submitted
163			"""
164			# add project info to this event
165			payload = add_project_info_dict(payload)
166
167			# submit list of events to Keen.io
168			keen.add_event(event_collection, payload)
169
170
171			def keen_add_events(event_collection, payload):
172			"""
173			Wrapper for keen.add_events(), adds project info to each event.
174
175			Param event_collection : collection event data is submitted to
176			Param payload : array of events that is submitted
177			"""
178			# add project info to each event
179			payload = add_project_info_list(payload)
180
181			# submit list of events to Keen.io
182			keen.add_events({event_collection: payload})
183
184
185			def add_project_info_dict(payload):
186			"""
187			Add project info to a dictonary.
188
189			Param payload: dictonary payload
190			"""
191			# check if payload is a dictionary, throws an exception if it isn't
192			check_dict(payload, "payload")
193
194			payload_as_dict = copy.deepcopy(payload)
195
196			payload_as_dict[KEEN_PROJECT_INFO_NAME] = Settings().get_project_info()
197
198			if "job" in payload:
199			# override project_name, set to build_job repo
200			if "repo" in payload["job"]:
201			payload_as_dict[KEEN_PROJECT_INFO_NAME]["project_name"] = \
202			payload["job"]["repo"]
203
204			# override timestamp, set to finished_at timestamp
205			if "finished_at" in payload["job"]:
206			payload_as_dict["keen"] = {
207			"timestamp": payload["job"]["finished_at"]["isotimestamp"]
208			}
209
210			return payload_as_dict
211
212
213			def add_project_info_list(payload):
214			"""
215			Add project info to a list of dictionaries.
216
217			Param payload: list of dictionaries
218			"""
219			# check if payload is a list, throws an exception if it isn't
220			is_list(payload, "payload")
221
222			payload_as_list = []
223
224			# loop over dicts in payload and add project info to each one
225			for event_dict in payload:
226			payload_as_list.append(add_project_info_dict(event_dict))
227
228			return payload_as_list
229
230
231			def get_dashboard_keen_config(repo):
232			"""
233			Generate the Keen.io settings for the configuration of the dashboard.
234
235			The dashboard is Javascript powered HTML file that contains the
236			graphs generated by Keen.io.
237			"""
238			# initialise config settings
239			keen_config = {}
240
241			if not keen_has_project_id() or not keen_has_master_key():
242			logger.warning("Keen.io related config settings could not be created,"
243			" keen.project_id and/or keen.master_key"
244			" are not defined.")
245			return keen_config
246
247			# set keen project ID
248			if keen.project_id is None:
249			keen_project_id = os.environ["KEEN_PROJECT_ID"]
250			else:
251			keen_project_id = keen.project_id
252			keen_config['projectId'] = str(keen_project_id)
253
254			# generate read key
255			read_key = keen_io_generate_read_key(repo)
256			if read_key is not None:
257			keen_config['readKey'] = str(read_key)
258
259			# return keen config settings
260			return keen_config
261
262
263			def get_dashboard_config(repo, extra=None):
264			"""
265			Generate the configuration settings for the dashboard.
266
267			The dashboard is Javascript powered HTML file that contains the
268			graphs generated by Keen.io.
269
270			Parameters:
271			- repo : repo name (fe. buildtimetrend/service)
272			- extra : dictionary of extra config settings, format : {"name" : "value"}
273			"""
274			# initialise config settings dictionaries
275			config = {}
276			keen_config = get_dashboard_keen_config(repo)
277
278			# add repo and project name
279			if repo is not None and not repo == "":
280			# merge extra settings into existing config dictionary
281			config.update({
282			'projectName': str(repo),
283			'repoName': str(repo)
284			})
285
286			# add extra config parameters
287			if extra is not None and check_dict(extra, "extra"):
288			config.update(extra)
289
290			# create configuration as a string
291			return "var config = %s;\nvar keenConfig = %s;" % (config, keen_config)
292
293
294			def generate_dashboard_config_file(repo):
295			"""
296			Generate the configuration file for the dashboard.
297
298			The dashboard is Javascript powered HTML file that contains the
299			graphs generated by Keen.io.
300
301			Parameters:
302			- repo : repo name (fe. buildtimetrend/service)
303			"""
304			# get config settings
305			config_string = get_dashboard_config(repo)
306
307			# write config file
308			config_file = Settings().get_setting("dashboard_configfile")
309			with open(config_file, 'w') as outfile:
310			outfile.write(config_string)
311
312			if check_file(config_file):
313			logger.info("Created trends dashboard config file %s", config_file)
314			return True
315			else:
316			logger.warning("The dashboard config file was not created")
317			return False
318
319
320			def get_avg_buildtime(repo=None, interval=None):
			0 ignored issues – show Duplication introduced 2015-11-22 12:44 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
321			"""
322			Query Keen.io database and retrieve average build time.
323
324			Parameters :
325			- repo : repo name (fe. buildtimetrend/service)
326			- interval : timeframe, possible values : 'week', 'month', 'year',
327			anything else defaults to 'week'
328			"""
329			if repo is None or not keen_is_readable():
330			return -1
331
332			interval_data = check_time_interval(interval)
333
334			try:
335			return keen.average(
336			"build_jobs",
337			target_property="job.duration",
338			timeframe=interval_data['timeframe'],
339			max_age=interval_data['max_age'],
340			filters=[get_repo_filter(repo)]
341			)
342			except requests.ConnectionError:
343			logger.error("Connection to Keen.io API failed")
344			return -1
345			except keen.exceptions.KeenApiError as msg:
346			logger.error("Error in keenio.get_avg_buildtime() : " + str(msg))
347			return -1
348
349
350			def get_total_build_jobs(repo=None, interval=None):
			0 ignored issues – show Duplication introduced 2015-11-22 12:44 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
351			"""
352			Query Keen.io database and retrieve total number of build jobs.
353
354			Parameters :
355			- repo : repo name (fe. buildtimetrend/service)
356			- interval : timeframe, possible values : 'week', 'month', 'year',
357			anything else defaults to 'week'
358			"""
359			if repo is None or not keen_is_readable():
360			return -1
361
362			interval_data = check_time_interval(interval)
363
364			try:
365			return keen.count_unique(
366			"build_jobs",
367			target_property="job.job",
368			timeframe=interval_data['timeframe'],
369			max_age=interval_data['max_age'],
370			filters=[get_repo_filter(repo)]
371			)
372			except requests.ConnectionError:
373			logger.error("Connection to Keen.io API failed")
374			return -1
375			except keen.exceptions.KeenApiError as msg:
376			logger.error("Error in keenio.get_total_build_jobs() : " + str(msg))
377			return -1
378
379
380			def get_passed_build_jobs(repo=None, interval=None):
381			"""
382			Query Keen.io database and retrieve total number of build jobs that passed.
383
384			Parameters :
385			- repo : repo name (fe. buildtimetrend/service)
386			- interval : timeframe, possible values : 'week', 'month', 'year',
387			anything else defaults to 'week'
388			"""
389			if repo is None or not keen_is_readable():
390			return -1
391
392			interval_data = check_time_interval(interval)
393
394			try:
395			return keen.count_unique(
396			"build_jobs",
397			target_property="job.job",
398			timeframe=interval_data['timeframe'],
399			max_age=interval_data['max_age'],
400			filters=[
401			get_repo_filter(repo),
402			{
403			"property_name": "job.result",
404			"operator": "eq",
405			"property_value": "passed"
406			}
407			]
408			)
409			except requests.ConnectionError:
410			logger.error("Connection to Keen.io API failed")
411			return -1
412			except keen.exceptions.KeenApiError as msg:
413			logger.error("Error in keenio.get_passed_build_jobs() : " + str(msg))
414			return -1
415
416
417			def get_pct_passed_build_jobs(repo=None, interval=None):
418			"""
419			Calculate percentage of passed build jobs.
420
421			Parameters :
422			- repo : repo name (fe. buildtimetrend/service)
423			- interval : timeframe, possible values : 'week', 'month', 'year',
424			anything else defaults to 'week'
425			"""
426			total_jobs = get_total_build_jobs(repo, interval)
427			passed_jobs = get_passed_build_jobs(repo, interval)
428
429			logger.debug("passed/total build jobs : %d/%d", passed_jobs, total_jobs)
430
431			# calculate percentage if at least one job was executed
432			# passed is a valid number (not -1)
433			if total_jobs > 0 and passed_jobs >= 0:
434			return int(float(passed_jobs) / float(total_jobs) * 100.0)
435
436			return -1
437
438
439			def get_result_color(value=0, ok_thershold=90, warning_thershold=70):
440			"""
441			Get color code that corresponds to the result.
442
443			Parameters:
444			- value : value to check
445			- ok_thershold : OK threshold value
446			- warning_thershold : warning thershold value
447			"""
448			if not(type(value) in (int, float) and
449			type(ok_thershold) in (int, float) and
450			type(warning_thershold) in (int, float)):
451			return "lightgrey"
452
453			# check thresholds
454			if value >= ok_thershold:
455			return "green"
456			elif value >= warning_thershold:
457			return "yellow"
458			else:
459			return "red"
460
461
462			def get_total_builds(repo=None, interval=None):
			0 ignored issues – show Duplication introduced 2015-11-22 12:44 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Duplicated code is one of the most pungent code smells. If you need to duplicate the same code in three or more different places, we strongly encourage you to look into extracting the code into a single class or operation. You can also find more detailed suggestions in the “Code” section of your repository. Loading history...
463			"""
464			Query Keen.io database and retrieve total number of builds.
465
466			Parameters :
467			- repo : repo name (fe. buildtimetrend/service)
468			- interval : timeframe, possible values : 'week', 'month', 'year',
469			anything else defaults to 'week'
470			"""
471			if repo is None or not keen_is_readable():
472			return -1
473
474			interval_data = check_time_interval(interval)
475
476			try:
477			return keen.count_unique(
478			"build_jobs",
479			target_property="job.build",
480			timeframe=interval_data['timeframe'],
481			max_age=interval_data['max_age'],
482			filters=[get_repo_filter(repo)]
483			)
484			except requests.ConnectionError:
485			logger.error("Connection to Keen.io API failed")
486			return -1
487			except keen.exceptions.KeenApiError as msg:
488			logger.error("Error in keenio.get_total_builds() : " + str(msg))
489			return -1
490
491
492			def get_latest_buildtime(repo=None):
493			"""
494			Query Keen.io database and retrieve buildtime duration of last build.
495
496			Parameters :
497			- repo : repo name (fe. buildtimetrend/python-lib)
498			"""
499			if repo is None or not keen_is_readable():
500			return -1
501
502			try:
503			result = keen.extraction(
504			"build_jobs",
505			property_names="job.duration",
506			latest=1,
507			filters=[get_repo_filter(repo)]
508			)
509			except requests.ConnectionError:
510			logger.error("Connection to Keen.io API failed")
511			return -1
512			except keen.exceptions.KeenApiError as msg:
513			logger.error("Error in keenio.get_latest_buildtime() : " + str(msg))
514			return -1
515
516			if result is not None and len(result) > 0:
517			return result[0]['job']['duration']
518
519			return -1
520
521
522			def has_build_id(repo=None, build_id=None):
523			"""
524			Check if build_id exists in Keen.io database.
525
526			Parameters :
527			- repo : repo name (fe. buildtimetrend/python-lib)
528			- build_id : ID of the build
529			"""
530			if repo is None or build_id is None:
531			logger.error("Repo or build_id is not set")
532			raise ValueError("Repo or build_id is not set")
533			if not keen_is_readable():
534			raise SystemError("Keen.io Project ID or API Read Key is not set")
535
536			try:
537			count = keen.count(
538			"build_jobs",
539			filters=[get_repo_filter(repo), {
540			"property_name": "job.build",
541			"operator": "eq",
542			"property_value": str(build_id)}]
543			)
544			return count > 0
545			except requests.ConnectionError:
546			logger.error("Connection to Keen.io API failed")
547			raise SystemError("Connection to Keen.io API failed")
548			except keen.exceptions.KeenApiError as msg:
549			logger.error("Error in keenio.has_build_id : " + str(msg))
550			raise SystemError(msg)
551
552
553			def get_all_projects():
554			"""Query Keen.io database and retrieve a list of all projects."""
555			if not keen_is_readable():
556			return []
557
558			try:
559			result = keen.select_unique(
560			"build_jobs",
561			"buildtime_trend.project_name",
562			max_age=3600 * 24 # cache for 24 hours
563			)
564			except requests.ConnectionError:
565			logger.error("Connection to Keen.io API failed")
566			return []
567			except keen.exceptions.KeenApiError as msg:
568			logger.error("Error in keenio.get_all_projects() : " + str(msg))
569			return -1
570
571			if type(result) is list:
572			return result
573
574			return []
575
576
577			def get_repo_filter(repo=None):
578			"""
579			Return filter for analysis request.
580
581			Parameters
582			- repo : repo slug name, fe. buildtimetrend/python-lib
583			"""
584			if repo is None:
585			return None
586
587			return {
588			"property_name": "%s.project_name" % KEEN_PROJECT_INFO_NAME,
589			"operator": "eq",
590			"property_value": str(repo)
591			}
592
593
594			def check_time_interval(interval=None):
595			"""
596			Check time interval and returns corresponding parameters.
597
598			Parameters :
599			- interval : timeframe, possible values : 'week', 'month', 'year',
600			anything else defaults to 'week'
601			"""
602			if is_string(interval):
603			# convert to lowercase
604			interval = interval.lower()
605
606			if interval in TIME_INTERVALS:
607			return TIME_INTERVALS[interval]
608
609			return TIME_INTERVALS['week']
610

buildtimetrend / python-lib

Push — master ( 44a554...4ba01b )

buildtimetrend.get_avg_buildtime() B

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like