fetch_job() - Code Metrics - troeger/opensubmit - Measure and Improve Code Quality continuously with Scrutinizer

fetch_job() F
last analyzed 2018-05-24 09:45 UTC

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
dl	0
loc	73
rs	3.4615
c	0
b	0
f	0
cc	11

How to fix Long Method Complexity

'''
Internal functions related to the communication with the
OpenSubmit server.
'''

import os
import shutil
import os.path
import glob
import json

from .exceptions import *
from .filesystem import *
from .hostinfo import ipaddress, all_host_infos

from urllib.request import urlopen, urlretrieve
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode

import logging
logger = logging.getLogger('opensubmitexec')


def fetch(url, fullpath):
    '''
    Fetch data from an URL and save it under the given target name.
    '''
    logger.debug("Fetching %s from %s" % (fullpath, url))

    tmpfile, headers = urlretrieve(url)
    if os.path.exists(fullpath):
        os.remove(fullpath)
    shutil.move(tmpfile, fullpath)


def send_post(config, urlpath, post_data):
    '''
    Send POST data to an OpenSubmit server url path,
    according to the configuration.
    '''
    server = config.get("Server", "url")
    post_data = urlencode(post_data)
    post_data = post_data.encode("utf-8", errors="ignore")
    url = server + urlpath
    try:
        urlopen(url, post_data)
    except Exception as e:
        logger.error('Error while sending data to server: ' + str(e))


def send_hostinfo(config):
    '''
    Register this host on OpenSubmit test machine.
    '''
    info = all_host_infos()
    logger.debug("Sending host information: " + str(info))
    post_data = [("Config", json.dumps(info)),
                 ("Action", "get_config"),
                 ("UUID", config.get("Server", "uuid")),
                 ("Address", ipaddress()),
                 ("Secret", config.get("Server", "secret"))
                 ]

    send_post(config, "/machines/", post_data)


def compatible_api_version(server_version):
    '''
    Check if this server API version is compatible to us.
    '''
    try:
        semver = server_version.split('.')
        if semver[0] != '1':
            logger.error(
                'Server API version (%s) is too new for us. Please update the executor installation.' % server_version)
            return False
        else:
            return True
    except Exception:
        logger.error(
            'Cannot understand the server API version (%s). Please update the executor installation.' % server_version)
        return False


def fetch_job(config):
    '''
    Fetch any available work from the OpenSubmit server and
    return an according job object.

    Returns None if no work is available.

    Errors are reported by this function directly.
    '''
    url = "%s/jobs/?Secret=%s&UUID=%s" % (config.get("Server", "url"),
                                          config.get("Server", "secret"),
                                          config.get("Server", "uuid"))

    try:
        # Fetch information from server
        result = urlopen(url)
        headers = result.info()
        if not compatible_api_version(headers["APIVersion"]):
            # No proper reporting possible, so only logging.
            logger.error("Incompatible API version. Please update OpenSubmit.")
            return None

        if headers["Action"] == "get_config":
            # The server does not know us,
            # so it demands registration before hand.
            logger.info("Machine unknown on server, sending registration ...")
            send_hostinfo(config)
            return None

        # Create job object with information we got
        from .job import Job
        job = Job(config)

        job.submitter_name = headers['SubmitterName']
        job.author_names = headers['AuthorNames']
        job.submitter_studyprogram = headers['SubmitterStudyProgram']
        job.course = headers['Course']
        job.assignment = headers['Assignment']
        job.action = headers["Action"]
        job.file_id = headers["SubmissionFileId"]
        job.sub_id = headers["SubmissionId"]
        job.file_name = headers["SubmissionOriginalFilename"]
        job.submitter_student_id = headers["SubmitterStudentId"]
        if "Timeout" in headers:
            job.timeout = int(headers["Timeout"])
        if "PostRunValidation" in headers:
            job.validator_url = headers["PostRunValidation"]
        job.working_dir = create_working_dir(config, job.sub_id)

        # Store submission in working directory
        submission_fname = job.working_dir + job.file_name
        with open(submission_fname, 'wb') as target:
            target.write(result.read())
        assert(os.path.exists(submission_fname))

        # Store validator package in working directory
        validator_fname = job.working_dir + 'download.validator'
        fetch(job.validator_url, validator_fname)

        try:
            prepare_working_directory(job, submission_fname, validator_fname)
        except JobException as e:
            job.send_fail_result(e.info_student, e.info_tutor)
            return None
        logger.debug("Got job: " + str(job))
        return job
    except HTTPError as e:
        if e.code == 404:
            logger.debug("Nothing to do.")
            return None
    except URLError as e:
        logger.error("Error while contacting {0}: {1}".format(url, str(e)))
        return None


def fake_fetch_job(config, src_dir):
    '''
    Act like fetch_job, but take the validator file and the student
    submission files directly from a directory.

    Intended for testing purposes when developing test scripts.

    Check also cmdline.py.
    '''
    logger.debug("Creating fake job from " + src_dir)
    from .job import Job
    job = Job(config, online=False)
    job.working_dir = create_working_dir(config, '42')
    for fname in glob.glob(src_dir + os.sep + '*'):
        logger.debug("Copying {0} to {1} ...".format(fname, job.working_dir))
        shutil.copy(fname, job.working_dir)
    case_files = glob.glob(job.working_dir + os.sep + '*')
    assert(len(case_files) == 2)
    if os.path.basename(case_files[0]) in ['validator.py', 'validator.zip']:
        validator = case_files[0]
        submission = case_files[1]
    else:
        validator = case_files[1]
        submission = case_files[0]
    logger.debug('{0} is the validator.'.format(validator))
    logger.debug('{0} the submission.'.format(submission))
    try:
        prepare_working_directory(job,
                                  submission_path=submission,
                                  validator_path=validator)
    except JobException as e:
        job.send_fail_result(e.info_student, e.info_tutor)
        return None
    logger.debug("Got fake job: " + str(job))
    return job


1			'''
2			Internal functions related to the communication with the
3			OpenSubmit server.
4			'''
5
6			import os
7			import shutil
8			import os.path
9			import glob
10			import json
11
12			from .exceptions import *
13			from .filesystem import *
14			from .hostinfo import ipaddress, all_host_infos
15
16			from urllib.request import urlopen, urlretrieve
17			from urllib.error import HTTPError, URLError
18			from urllib.parse import urlencode
19
20			import logging
21			logger = logging.getLogger('opensubmitexec')
22
23
24			def fetch(url, fullpath):
25			'''
26			Fetch data from an URL and save it under the given target name.
27			'''
28			logger.debug("Fetching %s from %s" % (fullpath, url))
29
30			tmpfile, headers = urlretrieve(url)
31			if os.path.exists(fullpath):
32			os.remove(fullpath)
33			shutil.move(tmpfile, fullpath)
34
35
36			def send_post(config, urlpath, post_data):
37			'''
38			Send POST data to an OpenSubmit server url path,
39			according to the configuration.
40			'''
41			server = config.get("Server", "url")
42			post_data = urlencode(post_data)
43			post_data = post_data.encode("utf-8", errors="ignore")
44			url = server + urlpath
45			try:
46			urlopen(url, post_data)
47			except Exception as e:
48			logger.error('Error while sending data to server: ' + str(e))
49
50
51			def send_hostinfo(config):
52			'''
53			Register this host on OpenSubmit test machine.
54			'''
55			info = all_host_infos()
56			logger.debug("Sending host information: " + str(info))
57			post_data = [("Config", json.dumps(info)),
58			("Action", "get_config"),
59			("UUID", config.get("Server", "uuid")),
60			("Address", ipaddress()),
61			("Secret", config.get("Server", "secret"))
62			]
63
64			send_post(config, "/machines/", post_data)
65
66
67			def compatible_api_version(server_version):
68			'''
69			Check if this server API version is compatible to us.
70			'''
71			try:
72			semver = server_version.split('.')
73			if semver[0] != '1':
74			logger.error(
75			'Server API version (%s) is too new for us. Please update the executor installation.' % server_version)
76			return False
77			else:
78			return True
79			except Exception:
80			logger.error(
81			'Cannot understand the server API version (%s). Please update the executor installation.' % server_version)
82			return False
83
84
85			def fetch_job(config):
86			'''
87			Fetch any available work from the OpenSubmit server and
88			return an according job object.
89
90			Returns None if no work is available.
91
92			Errors are reported by this function directly.
93			'''
94			url = "%s/jobs/?Secret=%s&UUID=%s" % (config.get("Server", "url"),
95			config.get("Server", "secret"),
96			config.get("Server", "uuid"))
97
98			try:
99			# Fetch information from server
100			result = urlopen(url)
101			headers = result.info()
102			if not compatible_api_version(headers["APIVersion"]):
103			# No proper reporting possible, so only logging.
104			logger.error("Incompatible API version. Please update OpenSubmit.")
105			return None
106
107			if headers["Action"] == "get_config":
108			# The server does not know us,
109			# so it demands registration before hand.
110			logger.info("Machine unknown on server, sending registration ...")
111			send_hostinfo(config)
112			return None
113
114			# Create job object with information we got
115			from .job import Job
116			job = Job(config)
117
118			job.submitter_name = headers['SubmitterName']
119			job.author_names = headers['AuthorNames']
120			job.submitter_studyprogram = headers['SubmitterStudyProgram']
121			job.course = headers['Course']
122			job.assignment = headers['Assignment']
123			job.action = headers["Action"]
124			job.file_id = headers["SubmissionFileId"]
125			job.sub_id = headers["SubmissionId"]
126			job.file_name = headers["SubmissionOriginalFilename"]
127			job.submitter_student_id = headers["SubmitterStudentId"]
128			if "Timeout" in headers:
129			job.timeout = int(headers["Timeout"])
130			if "PostRunValidation" in headers:
131			job.validator_url = headers["PostRunValidation"]
132			job.working_dir = create_working_dir(config, job.sub_id)
133
134			# Store submission in working directory
135			submission_fname = job.working_dir + job.file_name
136			with open(submission_fname, 'wb') as target:
137			target.write(result.read())
138			assert(os.path.exists(submission_fname))
139
140			# Store validator package in working directory
141			validator_fname = job.working_dir + 'download.validator'
142			fetch(job.validator_url, validator_fname)
143
144			try:
145			prepare_working_directory(job, submission_fname, validator_fname)
146			except JobException as e:
147			job.send_fail_result(e.info_student, e.info_tutor)
148			return None
149			logger.debug("Got job: " + str(job))
150			return job
151			except HTTPError as e:
152			if e.code == 404:
153			logger.debug("Nothing to do.")
154			return None
155			except URLError as e:
156			logger.error("Error while contacting {0}: {1}".format(url, str(e)))
157			return None
158
159
160			def fake_fetch_job(config, src_dir):
161			'''
162			Act like fetch_job, but take the validator file and the student
163			submission files directly from a directory.
164
165			Intended for testing purposes when developing test scripts.
166
167			Check also cmdline.py.
168			'''
169			logger.debug("Creating fake job from " + src_dir)
170			from .job import Job
171			job = Job(config, online=False)
172			job.working_dir = create_working_dir(config, '42')
173			for fname in glob.glob(src_dir + os.sep + '*'):
174			logger.debug("Copying {0} to {1} ...".format(fname, job.working_dir))
175			shutil.copy(fname, job.working_dir)
176			case_files = glob.glob(job.working_dir + os.sep + '*')
177			assert(len(case_files) == 2)
178			if os.path.basename(case_files[0]) in ['validator.py', 'validator.zip']:
179			validator = case_files[0]
180			submission = case_files[1]
181			else:
182			validator = case_files[1]
183			submission = case_files[0]
184			logger.debug('{0} is the validator.'.format(validator))
185			logger.debug('{0} the submission.'.format(submission))
186			try:
187			prepare_working_directory(job,
188			submission_path=submission,
189			validator_path=validator)
190			except JobException as e:
191			job.send_fail_result(e.info_student, e.info_tutor)
192			return None
193			logger.debug("Got fake job: " + str(job))
194			return job
195

troeger / opensubmit

fetch_job() F last analyzed 2018-05-24 09:45 UTC

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like

fetch_job() F
last analyzed 2018-05-24 09:45 UTC