send_post() - Code Metrics - Inspection of "Declare Python 3.4 compatibility" - troeger/opensubmit - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 531895...7cf997 )

by Peter

created 2017-11-30 21:57 UTC

send_post() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
c	1
b	0
f	0
dl	0
loc	13
rs	9.4285
cc	2

'''
The official executor API for validation test and full test scripts.
'''

import os
import sys
import importlib
import shutil
import os.path
import glob
import json

from .compiler import compiler_cmdline, GCC
from .config import read_config
from .running import kill_longrunning, RunningProgram
from .exceptions import *
from .filesystem import has_file, create_working_dir, prepare_working_directory
from .hostinfo import all_host_infos, ipaddress

from urllib.request import urlopen, urlretrieve
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode

import logging
logger = logging.getLogger('opensubmitexec')

UNSPECIFIC_ERROR = -9999


class Job():
    '''
    A OpenSubmit job to be run by the test machine.
    '''

    # The current executor configuration.
    _config = None
    # Talk to the configured OpenSubmit server?
    _online = None

    # Download source for the student sub
    submission_url = None
    # Download source for the validator
    validator_url = None
    # The working directory for this job
    working_dir = None
    # The timeout for execution, as demanded by the server
    timeout = None
    # The OpenSubmit submission ID
    submission_id = None
    # The OpenSubmit submission file ID
    file_id = None
    # Did the validator script sent a result to the server?
    result_sent = False
    # Action requested by the server (legacy)
    action = None
    # Name of the submitting student
    submitter_name = None
    # Student ID of the submitting student
    submitter_student_id = None
    # Names of the submission authors
    author_names = None
    # Name of the study program of the submitter
    submitter_studyprogram = None
    # Name of the course where this submission was done
    course = None
    # Name of the assignment where this job was done
    assignment = None

    # The base name of the validation / full test script
    # on disk, for importing.
    _validator_import_name = 'validator'

    @property
    # The file name of the validation / full test script
    # on disk, after unpacking / renaming.
    def validator_script_name(self):
        return self.working_dir + self._validator_import_name + '.py'

    def __init__(self, config=None, online=True):
        if config:
            self._config = config
        else:
            self._config = read_config()
        self._online = online

    def __str__(self):
        '''
        Nicer logging of job objects.
        '''
        return str(vars(self))

    def _run_validate(self):
        '''
        Execute the validate() method in the test script belonging to this job.
        '''
        assert(os.path.exists(self.validator_script_name))
        old_path = sys.path
        sys.path = [self.working_dir] + old_path
        # logger.debug('Python search path is now {0}.'.format(sys.path))
        module = importlib.import_module(self._validator_import_name)

        # Looped validator loading in the test suite demands this
        importlib.reload(module)

        # make the call
        try:
            module.validate(self)
        except Exception as e:
            # get more info
            text_student = None
            text_tutor = None
            if type(e) is TerminationException:
                text_student = "The execution of '{0}' terminated unexpectely.".format(
                    e.instance.name)
                text_tutor = "The execution of '{0}' terminated unexpectely.".format(
                    e.instance.name)
                text_student += "\n\nOutput so far:\n" + e.output
                text_tutor += "\n\nOutput so far:\n" + e.output
            elif type(e) is TimeoutException:
                text_student = "The execution of '{0}' was cancelled, since it took too long.".format(
                    e.instance.name)
                text_tutor = "The execution of '{0}' was cancelled due to timeout.".format(
                    e.instance.name)
                text_student += "\n\nOutput so far:\n" + e.output
                text_tutor += "\n\nOutput so far:\n" + e.output
            elif type(e) is NestedException:
                text_student = "Unexpected problem during the execution of '{0}'. {1}".format(
                    e.instance.name,
                    str(e.real_exception))
                text_tutor = "Unkown exception during the execution of '{0}'. {1}".format(
                    e.instance.name,
                    str(e.real_exception))
                text_student += "\n\nOutput so far:\n" + e.output
                text_tutor += "\n\nOutput so far:\n" + e.output
            elif type(e) is WrongExitStatusException:
                text_student = "The execution of '{0}' resulted in the unexpected exit status {1}.".format(
                    e.instance.name,
                    e.got)
                text_tutor = "The execution of '{0}' resulted in the unexpected exit status {1}.".format(
                    e.instance.name,
                    e.got)
                text_student += "\n\nOutput so far:\n" + e.output
                text_tutor += "\n\nOutput so far:\n" + e.output
            elif type(e) is JobException:
                # Some problem with our own code
                text_student = e.info_student
                text_tutor = e.info_tutor
            elif type(e) is FileNotFoundError:
                text_student = "A file is missing: {0}".format(
                    str(e))
                text_tutor = "Missing file: {0}".format(
                    str(e))
            elif type(e) is AssertionError:
                # Need this harsh approach to kill the
                # test suite execution at this point
                # Otherwise, the problem gets lost in
                # the log storm
                logger.error(
                    "Failed assertion in validation script. Should not happen in production.")
                exit(-1)
            else:
                # Something really unexpected
                text_student = "Internal problem while validating your submission. {0}".format(
                    str(e))
                text_tutor = "Unknown exception while running the validator. {0}".format(
                    str(e))
            # We got the text. Report the problem.
            self._send_result(text_student, text_tutor, UNSPECIFIC_ERROR)
            return
        # no unhandled exception during the execution of the validator
        if not self.result_sent:
            logger.debug("Validation script forgot result sending.")
            self.send_pass_result()
        # roll back
        sys.path = old_path

    def _send_result(self, info_student, info_tutor, error_code):
        post_data = [("SubmissionFileId", self.file_id),
                     ("Message", info_student),
                     ("Action", self.action),
                     ("MessageTutor", info_tutor),
                     ("ExecutorDir", self.working_dir),
                     ("ErrorCode", error_code),
                     ("Secret", self._config.get("Server", "secret")),
                     ("UUID", self._config.get("Server", "uuid"))
                     ]
        logger.info(
            'Sending result to OpenSubmit Server: ' + str(post_data))
        if self._online:
            send_post(self._config, "/jobs/", post_data)
        self.result_sent = True

    def send_fail_result(self, info_student, info_tutor):
        self._send_result(info_student, info_tutor, UNSPECIFIC_ERROR)

    def send_pass_result(self,
                         info_student="All tests passed. Awesome!",
                         info_tutor="All tests passed."):
        self._send_result(info_student, info_tutor, 0)

    def delete_binaries(self):
        '''
        Scans the submission files in the self.working_dir for
        binaries and deletes them.
        Returns the list of deleted files.
        '''
        raise NotImplementedError

    def run_configure(self, mandatory=True):
        '''
        Runs the configure tool configured for the machine in self.working_dir.
        '''
        if not has_file(self.working_dir, 'configure'):
            if mandatory:
                raise FileNotFoundError(
                    "Could not find a configure script for execution.")
            else:
                return
        try:
            prog = RunningProgram(self, 'configure')
            prog.expect_exit_status(0)
        except Exception:
            if mandatory:
                raise

    def run_make(self, mandatory=True):
        '''
        Runs the make tool configured for the machine in self.working_dir.
        '''
        if not has_file(self.working_dir, 'Makefile'):
            if mandatory:
                raise FileNotFoundError("Could not find a Makefile.")
            else:
                return
        try:
            prog = RunningProgram(self, 'make')
            prog.expect_exit_status(0)
        except Exception:
            if mandatory:
                raise

    def run_compiler(self, compiler=GCC, inputs=None, output=None):
        '''
        Runs the compiler in self.working_dir.
        '''
        # Let exceptions travel through
        prog = RunningProgram(self, *compiler_cmdline(compiler=compiler,
                                                      inputs=inputs,
                                                      output=output))
        prog.expect_exit_status(0)

    def run_build(self, compiler=GCC, inputs=None, output=None):
        logger.info("Running build steps ...")
        self.run_configure(mandatory=False)
        self.run_make(mandatory=False)
        self.run_compiler(compiler=compiler,
                          inputs=inputs,
                          output=output)

    def spawn_program(self, name, arguments=[], timeout=30, exclusive=False):
        '''
        Spawns a program in the working directory and allows
        interaction with it. Returns a RunningProgram object.

        The caller can demand exclusive execution on this machine.
        '''
        logger.debug("Spawning program for interaction ...")
        if exclusive:
            kill_longrunning(self.config)

        return RunningProgram(self, name, arguments, timeout)

    def run_program(self, name, arguments=[], timeout=30, exclusive=False):
        '''
        Runs a program in the working directory.
        The result is a tuple of exit code and output.

        The caller can demand exclusive execution on this machine.
        '''
        logger.debug("Running program ...")
        if exclusive:
            kill_longrunning(self.config)

        prog = RunningProgram(self, name, arguments, timeout)
        return prog.expect_end()

    def find_keywords(self, keywords, filepattern):
        '''
        Searches self.working_dir for files containing specific keywords.
        Expects a list of keywords to be searched for and the file pattern
        (*.c) as parameters.
        Returns the names of the files containing all of the keywords.
        '''
        raise NotImplementedError

    def ensure_files(self, filenames):
        '''
        Searches the student submission for specific files.
        Expects a list of filenames. Returns a boolean indicator.
        '''
        logger.debug("Testing {0} for the following files: {1}".format(
            self.working_dir, filenames))
        dircontent = os.listdir(self.working_dir)
        for fname in filenames:
            if fname not in dircontent:
                return False
        return True


def fetch(url, fullpath):
    '''
    Fetch data from an URL and save it under the given target name.
    '''
    logger.debug("Fetching %s from %s" % (fullpath, url))

    tmpfile, headers = urlretrieve(url)
    if os.path.exists(fullpath):
        os.remove(fullpath)
    shutil.move(tmpfile, fullpath)


def send_post(config, urlpath, post_data):
    '''
    Send POST data to an OpenSubmit server url path,
    according to the configuration.
    '''
    server = config.get("Server", "url")
    post_data = urlencode(post_data)
    post_data = post_data.encode("utf-8", errors="ignore")
    url = server + urlpath
    try:
        urlopen(url, post_data)
    except Exception as e:
        logger.error('Error while sending data to server: ' + str(e))


def send_hostinfo(config):
    '''
    Register this host on OpenSubmit test machine.
    '''
    info = all_host_infos()
    logger.debug("Sending host information: " + str(info))
    post_data = [("Config", json.dumps(info)),
                 ("Action", "get_config"),
                 ("UUID", config.get("Server", "uuid")),
                 ("Address", ipaddress()),
                 ("Secret", config.get("Server", "secret"))
                 ]

    send_post(config, "/machines/", post_data)


def compatible_api_version(server_version):
    '''
    Check if this server API version is compatible to us.
    '''
    try:
        semver = server_version.split('.')
        if semver[0] != '1':
            logger.error('Server API version (%s) is too new for us. Please update the executor installation.'%server_version)
            return False
        else:
            return True
    except Exception:
        logger.error('Cannot understand the server API version (%s). Please update the executor installation.'%server_version)
        return False


def fetch_job(config):
    '''
    Fetch any available work from the OpenSubmit server and
    return an according job object, or None.
    '''
    url = "%s/jobs/?Secret=%s&UUID=%s" % (config.get("Server", "url"),
                                          config.get("Server", "secret"),
                                          config.get("Server", "uuid"))

    try:
        # Fetch information from server
        result = urlopen(url)
        headers = result.info()
        if not compatible_api_version(headers["APIVersion"]):
            return None

        if headers["Action"] == "get_config":
            # The server does not know us,
            # so it demands registration before hand.
            logger.info("Machine unknown on server, sending registration ...")
            send_hostinfo(config)
            return None

        # Create job object with information we got
        job = Job(config)

        job.submitter_name = headers['SubmitterName']
        job.author_names = headers['AuthorNames']
        job.submitter_studyprogram = headers['SubmitterStudyProgram']
        job.course = headers['Course']
        job.assignment = headers['Assignment']
        job.action = headers["Action"]
        job.file_id = headers["SubmissionFileId"]
        job.sub_id = headers["SubmissionId"]
        job.file_name = headers["SubmissionOriginalFilename"]
        job.submitter_student_id = headers["SubmitterStudentId"]
        if "Timeout" in headers:
            job.timeout = int(headers["Timeout"])
        if "PostRunValidation" in headers:
            job.validator_url = headers["PostRunValidation"]
        job.working_dir = create_working_dir(config, job.sub_id)

        # Store submission in working directory
        submission_fname = job.working_dir + job.file_name
        with open(submission_fname, 'wb') as target:
            target.write(result.read())
        assert(os.path.exists(submission_fname))

        # Store validator package in working directory
        validator_fname = job.working_dir + 'download.validator'
        fetch(job.validator_url, validator_fname)

        try:
            prepare_working_directory(job, submission_fname, validator_fname)
        except JobException:
            logger.error("Preparation of working directory failed.")
            return None
        else:
            logger.debug("Got job: " + str(job))
            return job
    except HTTPError as e:
        if e.code == 404:
            logger.debug("Nothing to do.")
            return None
    except URLError as e:
        logger.error("Error while contacting {0}: {1}".format(url, str(e)))
        return None


def fake_fetch_job(config, src_dir):
    '''
    Act like fetch_job, but take the validator file and the student
    submission files directly from a directory.

    Intended for testing purposes when developing test scripts.

    Check also cmdline.py.
    '''
    logger.debug("Creating fake job from " + src_dir)
    job = Job(config, online=False)
    job.working_dir = create_working_dir(config, '42')
    case_files = glob.glob(src_dir + os.sep + '*')
    assert(len(case_files) == 2)
    for fname in glob.glob(src_dir + os.sep + '*'):
        logger.debug("Copying {0} to {1} ...".format(fname, job.working_dir))
        shutil.copy(fname, job.working_dir)
    if os.path.basename(case_files[0]) in ['validator.py', 'validator.zip']:
        validator = case_files[0]
        submission = case_files[1]
    else:
        validator = case_files[1]
        submission = case_files[0]
    logger.debug('{0} is the validator.'.format(validator))
    logger.debug('{0} the submission.'.format(submission))
    try:
        prepare_working_directory(job,
                                  submission_fname=submission,
                                  validator_fname=validator)
    except JobException:
        return None
    else:
        logger.debug("Got fake job: " + str(job))
        return job



1			'''
2			The official executor API for validation test and full test scripts.
3			'''
4
5			import os
6			import sys
7			import importlib
8			import shutil
9			import os.path
10			import glob
11			import json
12
13			from .compiler import compiler_cmdline, GCC
14			from .config import read_config
15			from .running import kill_longrunning, RunningProgram
16			from .exceptions import *
17			from .filesystem import has_file, create_working_dir, prepare_working_directory
18			from .hostinfo import all_host_infos, ipaddress
19
20			from urllib.request import urlopen, urlretrieve
21			from urllib.error import HTTPError, URLError
22			from urllib.parse import urlencode
23
24			import logging
25			logger = logging.getLogger('opensubmitexec')
26
27			UNSPECIFIC_ERROR = -9999
28
29
30			class Job():
31			'''
32			A OpenSubmit job to be run by the test machine.
33			'''
34
35			# The current executor configuration.
36			_config = None
37			# Talk to the configured OpenSubmit server?
38			_online = None
39
40			# Download source for the student sub
41			submission_url = None
42			# Download source for the validator
43			validator_url = None
44			# The working directory for this job
45			working_dir = None
46			# The timeout for execution, as demanded by the server
47			timeout = None
48			# The OpenSubmit submission ID
49			submission_id = None
50			# The OpenSubmit submission file ID
51			file_id = None
52			# Did the validator script sent a result to the server?
53			result_sent = False
54			# Action requested by the server (legacy)
55			action = None
56			# Name of the submitting student
57			submitter_name = None
58			# Student ID of the submitting student
59			submitter_student_id = None
60			# Names of the submission authors
61			author_names = None
62			# Name of the study program of the submitter
63			submitter_studyprogram = None
64			# Name of the course where this submission was done
65			course = None
66			# Name of the assignment where this job was done
67			assignment = None
68
69			# The base name of the validation / full test script
70			# on disk, for importing.
71			_validator_import_name = 'validator'
72
73			@property
74			# The file name of the validation / full test script
75			# on disk, after unpacking / renaming.
76			def validator_script_name(self):
77			return self.working_dir + self._validator_import_name + '.py'
78
79			def __init__(self, config=None, online=True):
80			if config:
81			self._config = config
82			else:
83			self._config = read_config()
84			self._online = online
85
86			def __str__(self):
87			'''
88			Nicer logging of job objects.
89			'''
90			return str(vars(self))
91
92			def _run_validate(self):
93			'''
94			Execute the validate() method in the test script belonging to this job.
95			'''
96			assert(os.path.exists(self.validator_script_name))
97			old_path = sys.path
98			sys.path = [self.working_dir] + old_path
99			# logger.debug('Python search path is now {0}.'.format(sys.path))
100			module = importlib.import_module(self._validator_import_name)
101
102			# Looped validator loading in the test suite demands this
103			importlib.reload(module)
104
105			# make the call
106			try:
107			module.validate(self)
108			except Exception as e:
109			# get more info
110			text_student = None
111			text_tutor = None
112			if type(e) is TerminationException:
113			text_student = "The execution of '{0}' terminated unexpectely.".format(
114			e.instance.name)
115			text_tutor = "The execution of '{0}' terminated unexpectely.".format(
116			e.instance.name)
117			text_student += "\n\nOutput so far:\n" + e.output
118			text_tutor += "\n\nOutput so far:\n" + e.output
119			elif type(e) is TimeoutException:
120			text_student = "The execution of '{0}' was cancelled, since it took too long.".format(
121			e.instance.name)
122			text_tutor = "The execution of '{0}' was cancelled due to timeout.".format(
123			e.instance.name)
124			text_student += "\n\nOutput so far:\n" + e.output
125			text_tutor += "\n\nOutput so far:\n" + e.output
126			elif type(e) is NestedException:
127			text_student = "Unexpected problem during the execution of '{0}'. {1}".format(
128			e.instance.name,
129			str(e.real_exception))
130			text_tutor = "Unkown exception during the execution of '{0}'. {1}".format(
131			e.instance.name,
132			str(e.real_exception))
133			text_student += "\n\nOutput so far:\n" + e.output
134			text_tutor += "\n\nOutput so far:\n" + e.output
135			elif type(e) is WrongExitStatusException:
136			text_student = "The execution of '{0}' resulted in the unexpected exit status {1}.".format(
137			e.instance.name,
138			e.got)
139			text_tutor = "The execution of '{0}' resulted in the unexpected exit status {1}.".format(
140			e.instance.name,
141			e.got)
142			text_student += "\n\nOutput so far:\n" + e.output
143			text_tutor += "\n\nOutput so far:\n" + e.output
144			elif type(e) is JobException:
145			# Some problem with our own code
146			text_student = e.info_student
147			text_tutor = e.info_tutor
148			elif type(e) is FileNotFoundError:
149			text_student = "A file is missing: {0}".format(
150			str(e))
151			text_tutor = "Missing file: {0}".format(
152			str(e))
153			elif type(e) is AssertionError:
154			# Need this harsh approach to kill the
155			# test suite execution at this point
156			# Otherwise, the problem gets lost in
157			# the log storm
158			logger.error(
159			"Failed assertion in validation script. Should not happen in production.")
160			exit(-1)
161			else:
162			# Something really unexpected
163			text_student = "Internal problem while validating your submission. {0}".format(
164			str(e))
165			text_tutor = "Unknown exception while running the validator. {0}".format(
166			str(e))
167			# We got the text. Report the problem.
168			self._send_result(text_student, text_tutor, UNSPECIFIC_ERROR)
169			return
170			# no unhandled exception during the execution of the validator
171			if not self.result_sent:
172			logger.debug("Validation script forgot result sending.")
173			self.send_pass_result()
174			# roll back
175			sys.path = old_path
176
177			def _send_result(self, info_student, info_tutor, error_code):
178			post_data = [("SubmissionFileId", self.file_id),
179			("Message", info_student),
180			("Action", self.action),
181			("MessageTutor", info_tutor),
182			("ExecutorDir", self.working_dir),
183			("ErrorCode", error_code),
184			("Secret", self._config.get("Server", "secret")),
185			("UUID", self._config.get("Server", "uuid"))
186			]
187			logger.info(
188			'Sending result to OpenSubmit Server: ' + str(post_data))
189			if self._online:
190			send_post(self._config, "/jobs/", post_data)
191			self.result_sent = True
192
193			def send_fail_result(self, info_student, info_tutor):
194			self._send_result(info_student, info_tutor, UNSPECIFIC_ERROR)
195
196			def send_pass_result(self,
197			info_student="All tests passed. Awesome!",
198			info_tutor="All tests passed."):
199			self._send_result(info_student, info_tutor, 0)
200
201			def delete_binaries(self):
202			'''
203			Scans the submission files in the self.working_dir for
204			binaries and deletes them.
205			Returns the list of deleted files.
206			'''
207			raise NotImplementedError
208
209			def run_configure(self, mandatory=True):
210			'''
211			Runs the configure tool configured for the machine in self.working_dir.
212			'''
213			if not has_file(self.working_dir, 'configure'):
214			if mandatory:
215			raise FileNotFoundError(
216			"Could not find a configure script for execution.")
217			else:
218			return
219			try:
220			prog = RunningProgram(self, 'configure')
221			prog.expect_exit_status(0)
222			except Exception:
223			if mandatory:
224			raise
225
226			def run_make(self, mandatory=True):
227			'''
228			Runs the make tool configured for the machine in self.working_dir.
229			'''
230			if not has_file(self.working_dir, 'Makefile'):
231			if mandatory:
232			raise FileNotFoundError("Could not find a Makefile.")
233			else:
234			return
235			try:
236			prog = RunningProgram(self, 'make')
237			prog.expect_exit_status(0)
238			except Exception:
239			if mandatory:
240			raise
241
242			def run_compiler(self, compiler=GCC, inputs=None, output=None):
243			'''
244			Runs the compiler in self.working_dir.
245			'''
246			# Let exceptions travel through
247			prog = RunningProgram(self, *compiler_cmdline(compiler=compiler,
248			inputs=inputs,
249			output=output))
250			prog.expect_exit_status(0)
251
252			def run_build(self, compiler=GCC, inputs=None, output=None):
253			logger.info("Running build steps ...")
254			self.run_configure(mandatory=False)
255			self.run_make(mandatory=False)
256			self.run_compiler(compiler=compiler,
257			inputs=inputs,
258			output=output)
259
260			def spawn_program(self, name, arguments=[], timeout=30, exclusive=False):
261			'''
262			Spawns a program in the working directory and allows
263			interaction with it. Returns a RunningProgram object.
264
265			The caller can demand exclusive execution on this machine.
266			'''
267			logger.debug("Spawning program for interaction ...")
268			if exclusive:
269			kill_longrunning(self.config)
270
271			return RunningProgram(self, name, arguments, timeout)
272
273			def run_program(self, name, arguments=[], timeout=30, exclusive=False):
274			'''
275			Runs a program in the working directory.
276			The result is a tuple of exit code and output.
277
278			The caller can demand exclusive execution on this machine.
279			'''
280			logger.debug("Running program ...")
281			if exclusive:
282			kill_longrunning(self.config)
283
284			prog = RunningProgram(self, name, arguments, timeout)
285			return prog.expect_end()
286
287			def find_keywords(self, keywords, filepattern):
288			'''
289			Searches self.working_dir for files containing specific keywords.
290			Expects a list of keywords to be searched for and the file pattern
291			(*.c) as parameters.
292			Returns the names of the files containing all of the keywords.
293			'''
294			raise NotImplementedError
295
296			def ensure_files(self, filenames):
297			'''
298			Searches the student submission for specific files.
299			Expects a list of filenames. Returns a boolean indicator.
300			'''
301			logger.debug("Testing {0} for the following files: {1}".format(
302			self.working_dir, filenames))
303			dircontent = os.listdir(self.working_dir)
304			for fname in filenames:
305			if fname not in dircontent:
306			return False
307			return True
308
309
310			def fetch(url, fullpath):
311			'''
312			Fetch data from an URL and save it under the given target name.
313			'''
314			logger.debug("Fetching %s from %s" % (fullpath, url))
315
316			tmpfile, headers = urlretrieve(url)
317			if os.path.exists(fullpath):
318			os.remove(fullpath)
319			shutil.move(tmpfile, fullpath)
320
321
322			def send_post(config, urlpath, post_data):
323			'''
324			Send POST data to an OpenSubmit server url path,
325			according to the configuration.
326			'''
327			server = config.get("Server", "url")
328			post_data = urlencode(post_data)
329			post_data = post_data.encode("utf-8", errors="ignore")
330			url = server + urlpath
331			try:
332			urlopen(url, post_data)
333			except Exception as e:
334			logger.error('Error while sending data to server: ' + str(e))
335
336
337			def send_hostinfo(config):
338			'''
339			Register this host on OpenSubmit test machine.
340			'''
341			info = all_host_infos()
342			logger.debug("Sending host information: " + str(info))
343			post_data = [("Config", json.dumps(info)),
344			("Action", "get_config"),
345			("UUID", config.get("Server", "uuid")),
346			("Address", ipaddress()),
347			("Secret", config.get("Server", "secret"))
348			]
349
350			send_post(config, "/machines/", post_data)
351
352
353			def compatible_api_version(server_version):
354			'''
355			Check if this server API version is compatible to us.
356			'''
357			try:
358			semver = server_version.split('.')
359			if semver[0] != '1':
360			logger.error('Server API version (%s) is too new for us. Please update the executor installation.'%server_version)
361			return False
362			else:
363			return True
364			except Exception:
365			logger.error('Cannot understand the server API version (%s). Please update the executor installation.'%server_version)
366			return False
367
368
369			def fetch_job(config):
370			'''
371			Fetch any available work from the OpenSubmit server and
372			return an according job object, or None.
373			'''
374			url = "%s/jobs/?Secret=%s&UUID=%s" % (config.get("Server", "url"),
375			config.get("Server", "secret"),
376			config.get("Server", "uuid"))
377
378			try:
379			# Fetch information from server
380			result = urlopen(url)
381			headers = result.info()
382			if not compatible_api_version(headers["APIVersion"]):
383			return None
384
385			if headers["Action"] == "get_config":
386			# The server does not know us,
387			# so it demands registration before hand.
388			logger.info("Machine unknown on server, sending registration ...")
389			send_hostinfo(config)
390			return None
391
392			# Create job object with information we got
393			job = Job(config)
394
395			job.submitter_name = headers['SubmitterName']
396			job.author_names = headers['AuthorNames']
397			job.submitter_studyprogram = headers['SubmitterStudyProgram']
398			job.course = headers['Course']
399			job.assignment = headers['Assignment']
400			job.action = headers["Action"]
401			job.file_id = headers["SubmissionFileId"]
402			job.sub_id = headers["SubmissionId"]
403			job.file_name = headers["SubmissionOriginalFilename"]
404			job.submitter_student_id = headers["SubmitterStudentId"]
405			if "Timeout" in headers:
406			job.timeout = int(headers["Timeout"])
407			if "PostRunValidation" in headers:
408			job.validator_url = headers["PostRunValidation"]
409			job.working_dir = create_working_dir(config, job.sub_id)
410
411			# Store submission in working directory
412			submission_fname = job.working_dir + job.file_name
413			with open(submission_fname, 'wb') as target:
414			target.write(result.read())
415			assert(os.path.exists(submission_fname))
416
417			# Store validator package in working directory
418			validator_fname = job.working_dir + 'download.validator'
419			fetch(job.validator_url, validator_fname)
420
421			try:
422			prepare_working_directory(job, submission_fname, validator_fname)
423			except JobException:
424			logger.error("Preparation of working directory failed.")
425			return None
426			else:
427			logger.debug("Got job: " + str(job))
428			return job
429			except HTTPError as e:
430			if e.code == 404:
431			logger.debug("Nothing to do.")
432			return None
433			except URLError as e:
434			logger.error("Error while contacting {0}: {1}".format(url, str(e)))
435			return None
436
437
438			def fake_fetch_job(config, src_dir):
439			'''
440			Act like fetch_job, but take the validator file and the student
441			submission files directly from a directory.
442
443			Intended for testing purposes when developing test scripts.
444
445			Check also cmdline.py.
446			'''
447			logger.debug("Creating fake job from " + src_dir)
448			job = Job(config, online=False)
449			job.working_dir = create_working_dir(config, '42')
450			case_files = glob.glob(src_dir + os.sep + '*')
451			assert(len(case_files) == 2)
452			for fname in glob.glob(src_dir + os.sep + '*'):
453			logger.debug("Copying {0} to {1} ...".format(fname, job.working_dir))
454			shutil.copy(fname, job.working_dir)
455			if os.path.basename(case_files[0]) in ['validator.py', 'validator.zip']:
456			validator = case_files[0]
457			submission = case_files[1]
458			else:
459			validator = case_files[1]
460			submission = case_files[0]
461			logger.debug('{0} is the validator.'.format(validator))
462			logger.debug('{0} the submission.'.format(submission))
463			try:
464			prepare_working_directory(job,
465			submission_fname=submission,
466			validator_fname=validator)
467			except JobException:
468			return None
469			else:
470			logger.debug("Got fake job: " + str(job))
471			return job
472
473

troeger / opensubmit

Push — master ( 531895...7cf997 )

send_post() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like