Communication.test_too_long_validation() - Code Metrics - Inspection of "Better handling of invalid validators. Closes #197" - troeger/opensubmit - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 7d61bb...58af22 )

by Peter

created 2017-12-16 11:33 UTC

Communication.test_too_long_validation() B

↳ Parent: Communication

Complexity

Conditions

Size

Total Lines

Duplication

Lines	27
Ratio	100 %

Importance

Changes	4
Bugs	0	Features	0

Metric	Value
c	4
b	0
f	0
dl	27
loc	27
rs	8.8571
cc	2

'''
    Test cases for the executor code.

    Note the neccessary python search path manipulation below.

Missing tests:

  - Makefile in validator package should override student Makefile

'''

import os
import os.path
import sys
import logging

from django.core import mail
from django.conf import settings
from django.test import TestCase
from django.test.utils import override_settings
from opensubmit.tests.cases import SubmitStudentScenarioTestCase
from django.core.urlresolvers import reverse

from opensubmit.models import TestMachine, SubmissionTestResult, Submission
from opensubmit.tests import utils

from .helpers.submission import create_validatable_submission
from .helpers.submission import create_validated_submission
from .helpers.assignment import create_validated_assignment
from .helpers.assignment import create_pass_fail_grading
from .helpers.assignment import create_validated_assignment_with_file
from .helpers.djangofiles import create_submission_file
from .helpers.user import create_user, get_student_dict
from . import uccrap, rootdir

sys.path.insert(0, os.path.dirname(__file__) + '/../../../executor/')
from opensubmitexec import config, cmdline, server, locking, compiler, exceptions  # NOQA

logger = logging.getLogger('opensubmitexec')



class CmdLine(TestCase):
    '''
    Test cases for the executor command-line script.
    '''
    def setUp(self):
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg")

    def test_help_call_without_config(self):
        # simulate 'help' call
        sys.argv = ['opensubmit-exec', 'help']
        cmdline.console_script()

    def test_unlock_call_without_lock(self):
        # simulate 'unlock' call without existing lock
        sys.argv = ['opensubmit-exec', 'unlock']
        cmdline.console_script()

    def test_unlock_call_with_lock(self):
        # simulate 'unlock' call with existing lock
        sys.argv = ['opensubmit-exec', 'unlock']
        with locking.ScriptLock(self.config):
            cmdline.console_script()


class Library(SubmitStudentScenarioTestCase):
    '''
    Tests for the executor library functions used by the validator script.
    '''
    def setUp(self):
        settings.MAIN_URL = self.live_server_url
        super(Library, self).setUp()
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg",
            override_url=self.live_server_url)

    def _register_executor(self):
        server.send_hostinfo(self.config)
        return TestMachine.objects.order_by('-last_contact')[0]

    def _run_executor(self):
        return cmdline.download_and_run(self.config)

    def test_send_result(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)

        msg = uccrap + "Message from validation script."

        job.send_pass_result(info_student=msg)

        db_entries = SubmissionTestResult.objects.filter(
            kind=SubmissionTestResult.VALIDITY_TEST)

        self.assertEqual(db_entries[0].result, msg)

    def test_fetch_job_renaming(self):
        test_machine = self._register_executor()
        self.validated_assignment.test_machines.add(test_machine)
        with open(rootdir + "submfiles/validation/1000fff/helloworld.c", 'rb') as f:
            self.c.post('/assignments/%s/new/' % self.validated_assignment.pk, {
                'notes': 'This is a test submission.',
                'authors': str(self.user.pk),
                'attachment': f
            })
        job = server.fetch_job(self.config)
        self.assertIn('helloworld.c', os.listdir(job.working_dir))

    def test_job_attributes(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)
        self.assertEquals(job.timeout, self.validated_assignment.attachment_test_timeout)
        self.assertEquals(job.sub_id, str(sub.pk))
        self.assertEquals(job.file_id, str(sf.pk))
        self.assertEquals(job.submitter_name, sub.submitter.get_full_name())
        self.assertEquals(job.submitter_student_id, str(sub.submitter.profile.student_id))
        self.assertEquals(job.submitter_studyprogram, str(sub.submitter.profile.study_program))
        self.assertEquals(job.course, str(self.course))
        self.assertEquals(job.assignment, str(self.validated_assignment))

    def test_wrong_compile_call(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        # Missing output
        self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, inputs=['input.c'])
        # Missing input
        self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, output='output')
        # Wrong file, determined by compiler
        self.assertRaises(exceptions.WrongExitStatusException, job.run_compiler, inputs=['foo.c'], output='output')


class Validation(TestCase):
    '''
    Tests for the execution of validation scripts by the executor.
    '''

    def setUp(self):
        super(Validation, self).setUp()
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg")

    def _test_validation_case(self, directory):
        '''
        Each of the validator.py files uses the Python assert()
        statement to check by itself if the result is the expected
        one for its case.
        '''
        base_dir = os.path.dirname(__file__) + '/submfiles/validation/'
        cmdline.copy_and_run(self.config, base_dir + directory)

    def test_0100fff(self):
        self._test_validation_case('0100fff')

    def test_0100tff(self):
        self._test_validation_case('0100tff')

    def test_0100ttf(self):
        self._test_validation_case('0100ttf')

    def test_1000fff(self):
        self._test_validation_case('1000fff')

    def test_1000fft(self):
        self._test_validation_case('1000fft')

    def test_1000tff(self):
        self._test_validation_case('1000tff')

    def test_1000tft(self):
        self._test_validation_case('1000tft')

    def test_1000ttf(self):
        self._test_validation_case('1000ttf')

    def test_1000ttt(self):
        self._test_validation_case('1000ttt')

    def test_1010tff(self):
        self._test_validation_case('1010tff')

    def test_1010ttf(self):
        self._test_validation_case('1010ttf')

    def test_1100tff(self):
        self._test_validation_case('1100tff')

    def test_1100ttf(self):
        self._test_validation_case('1100ttf')

    def test_3000tff(self):
        self._test_validation_case('3000tff')

    def test_3000ttf(self):
        self._test_validation_case('3000ttf')

    def test_3010tff(self):
        self._test_validation_case('3010tff')

    def test_3010ttf(self):
        self._test_validation_case('3010ttf')

    def test_b000tff(self):
        self._test_validation_case('b000tff')

    def test_b010tff(self):
        self._test_validation_case('b010tff')

    def test_1000tfm(self):
        self._test_validation_case('1000tfm')


class Communication(SubmitStudentScenarioTestCase):
    '''
    Tests for the communication of the executor with the OpenSubmit server.
    '''

    def setUp(self):
        settings.MAIN_URL = self.live_server_url
        super(Communication, self).setUp()
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg",
            override_url=self.live_server_url)

    def _register_executor(self):
        server.send_hostinfo(self.config)
        return TestMachine.objects.order_by('-last_contact')[0]

    def _run_executor(self):
        return cmdline.download_and_run(self.config)

    def _register_test_machine(self):
        '''
        Utility step for a common test case preparation:
        - Create validatable submission
        - Register a test machine for it
        '''
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        return sub

    def test_register_executor_explicit(self):
        machine_count = TestMachine.objects.all().count()
        assert(self._register_executor().pk)
        self.assertEqual(machine_count + 1, TestMachine.objects.all().count())

    @override_settings(JOB_EXECUTOR_SECRET='foo')
    def test_invalid_secret(self):
        self.assertNotEqual(True, self._run_executor())

    def test_everything_already_tested(self):
        create_validated_submission(self.user, self.validated_assignment)
        assert(self._register_executor().pk)
        self.assertEqual(False, self._run_executor())

    def test_parallel_executors_test(self):
        NUM_PARALLEL = 3
        self.validated_assignment.test_machines.add(self._register_executor())
        subs = []
        for i in range(1, NUM_PARALLEL + 1):
            stud = create_user(get_student_dict(i))
            self.course.participants.add(stud.profile)
            self.course.save()
            sf = create_submission_file()
            subs.append(create_validatable_submission(
                stud, self.validated_assignment, sf))

        # Span a number of threads, each triggering the executor
        # This only creates a real test case if executor serialization
        # is off (see tests/executor.cfg)
        return_codes = utils.run_parallel(len(subs), self._run_executor)
        self.assertEqual(
            len(list(filter((lambda x: x is True), return_codes))),
            len(subs))

        for sub in subs:
            results = SubmissionTestResult.objects.filter(
                kind=SubmissionTestResult.VALIDITY_TEST
            )
            self.assertEqual(NUM_PARALLEL, len(results))
            self.assertNotEqual(0, len(results[0].result))

    def test_too_long_validation(self):

        from django.core import mail

        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, grading, "/submfiles/validation/d000fff/", "validator_run.py")
        assignment.attachment_test_timeout = 1
        assignment.save()
        sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        # Fire up the executor, should mark the submission as timed out
        self.assertEqual(True, self._run_executor())
        # Check if timeout marking took place
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
        text = sub.get_validation_result().result
        self.assertIn("took too long", text)
        # Check mail outbox for student information
        self.assertEqual(1, len(mail.outbox))
        for email in mail.outbox:
            self.assertIn("Validation failed", email.subject)
            self.assertIn("failed", email.body)
            self.assertIn("localhost", email.body)

    def test_broken_validator_feedback(self):

        from django.core import mail

        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, grading, "/submfiles/validation/1000tfm/", "validator.zip")
        assignment.save()
        sf = create_submission_file("/submfiles/validation/1000tfm/packed.zip")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        # Fire up the executor
        self.assertEqual(False, self._run_executor())
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
        text = sub.get_validation_result().result
        self.assertIn("Internal error", text)
        # Check mail outbox for student information
        self.assertEqual(1, len(mail.outbox))
        for email in mail.outbox:
            self.assertIn("Validation failed", email.subject)
            self.assertIn("failed", email.body)
            self.assertIn("localhost", email.body)

    def test_too_long_full_test(self):

        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, 
            grading, 
            "/submfiles/validation/d000fff/",
            "validator_build.py",
            "validator_run.py")
        assignment.attachment_test_timeout = 1
        assignment.save()
        sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        # Fire up the executor for validation
        self.assertEqual(True, self._run_executor())
        # Fire up the executor for full test
        self.assertEqual(True, self._run_executor())
        # Check if timeout marking took place
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.TEST_FULL_FAILED)
        assert("took too long" in sub.get_fulltest_result().result)
        # Failed full tests shall not be reported
        self.assertEqual(0, len(mail.outbox))

    def test_single_file_validator_test(self):
        sub = self._register_test_machine()

        self.assertEqual(True, self._run_executor())
        results = SubmissionTestResult.objects.filter(
            submission_file=sub.file_upload,
            kind=SubmissionTestResult.VALIDITY_TEST
        )
        self.assertEqual(1, len(results))

        self.assertEqual(True, self._run_executor())
        results = SubmissionTestResult.objects.filter(
            submission_file=sub.file_upload,
            kind=SubmissionTestResult.VALIDITY_TEST
        )
        self.assertEqual(1, len(results))
        self.assertNotEqual(0, len(results[0].result))
        # Graded assignment, so no mail at this stage
        self.assertEqual(0, len(mail.outbox))

    def test_ungraded_validation_email(self):

        from django.core import mail

        sf = create_submission_file()
        assign = create_validated_assignment_with_file(self.course, None)
        sub = create_validatable_submission(self.user, assign, sf)
        test_machine = self._register_executor()
        assign.test_machines.add(test_machine)
        # has no grading scheme, so it is not graded
        assert(not sub.assignment.is_graded())

        # Fire up the executor for validation
        self.assertEqual(True, self._run_executor())

        # Fire up the executor for full test
        self.assertEqual(True, self._run_executor())

        # Check mail outbox for student information
        self.assertEqual(1, len(mail.outbox))
        for email in mail.outbox:
            self.assertIn("Validation successful", email.subject)
            self.assertIn("passed", email.body)
            self.assertIn("localhost", email.body)

    def test_full_test(self):
        sub = self._register_test_machine()
        # validation test
        self.assertEqual(True, self._run_executor())
        # full test
        self.assertEqual(True, self._run_executor())
        results = SubmissionTestResult.objects.filter(
            submission_file=sub.file_upload,
            kind=SubmissionTestResult.FULL_TEST
        )
        self.assertEqual(1, len(results))
        self.assertNotEqual(0, len(results[0].result))

    def test_inconsistent_state_email(self):
        '''
        Test operator email on inconsistent state.
        Since executor execution and submission sending is one step,
        we need to mock the incoming invalid executor request.
        '''
        sf = create_submission_file()
        self.sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        self.sub.assignment.test_machines.add(test_machine)
        self.sub.state = Submission.TEST_FULL_PENDING
        self.sub.save()
        post_data = {'Secret': settings.JOB_EXECUTOR_SECRET,
                     'UUID': test_machine.host,
                     'Action': 'test_compile',
                     'SubmissionFileId': self.sub.file_upload.pk,
                     'PerfData': '',
                     'ErrorCode': 0,
                     'Message': 'In A Bottle'}
        self.c.post(reverse('jobs'), post_data)
        self.assertNotEqual(len(mail.outbox), 0)
        self.assertIn('Action reported by the executor: test_compile',
                      mail.outbox[0].message().as_string())

    def test_assignment_specific_test_machine(self):
        # Register two test machines T1 and T2
        real_machine = self._register_executor()
        fake_machine = TestMachine(host="127.0.0.2")
        fake_machine.save()
        # Assign each of them to a different assignment
        self.open_assignment.test_machines.add(real_machine)
        self.validated_assignment.test_machines.add(fake_machine)
        # Produce submission for the assignment linked to fake_machine
        sub1 = Submission(
            assignment=self.validated_assignment,
            submitter=self.user,
            state=Submission.TEST_VALIDITY_PENDING,
            file_upload=create_submission_file()
        )
        sub1.save()
        # Run real_machine executor, should not react on this submission
        old_sub1_state = sub1.state
        self.assertEqual(False, self._run_executor())
        # Make sure that submission object was not touched,
        # whatever the executor says
        sub1 = Submission.objects.get(pk=sub1.pk)
        self.assertEqual(old_sub1_state, sub1.state)


1		'''
2		Test cases for the executor code.
3
4		Note the neccessary python search path manipulation below.
5
6		Missing tests:
7
8		- Makefile in validator package should override student Makefile
9
10		'''
11
12		import os
13		import os.path
14		import sys
15		import logging
16
17		from django.core import mail
18		from django.conf import settings
19		from django.test import TestCase
20		from django.test.utils import override_settings
21		from opensubmit.tests.cases import SubmitStudentScenarioTestCase
22		from django.core.urlresolvers import reverse
23
24		from opensubmit.models import TestMachine, SubmissionTestResult, Submission
25		from opensubmit.tests import utils
26
27		from .helpers.submission import create_validatable_submission
28		from .helpers.submission import create_validated_submission
29		from .helpers.assignment import create_validated_assignment
30		from .helpers.assignment import create_pass_fail_grading
31		from .helpers.assignment import create_validated_assignment_with_file
32		from .helpers.djangofiles import create_submission_file
33		from .helpers.user import create_user, get_student_dict
34		from . import uccrap, rootdir
35
36		sys.path.insert(0, os.path.dirname(__file__) + '/../../../executor/')
37		from opensubmitexec import config, cmdline, server, locking, compiler, exceptions # NOQA
38
39		logger = logging.getLogger('opensubmitexec')
40
41
42
43		class CmdLine(TestCase):
44		'''
45		Test cases for the executor command-line script.
46		'''
47		def setUp(self):
48		self.config = config.read_config(
49		os.path.dirname(__file__) + "/executor.cfg")
50
51		def test_help_call_without_config(self):
52		# simulate 'help' call
53		sys.argv = ['opensubmit-exec', 'help']
54		cmdline.console_script()
55
56		def test_unlock_call_without_lock(self):
57		# simulate 'unlock' call without existing lock
58		sys.argv = ['opensubmit-exec', 'unlock']
59		cmdline.console_script()
60
61		def test_unlock_call_with_lock(self):
62		# simulate 'unlock' call with existing lock
63		sys.argv = ['opensubmit-exec', 'unlock']
64		with locking.ScriptLock(self.config):
65		cmdline.console_script()
66
67
68		class Library(SubmitStudentScenarioTestCase):
69		'''
70		Tests for the executor library functions used by the validator script.
71		'''
72		def setUp(self):
73		settings.MAIN_URL = self.live_server_url
74		super(Library, self).setUp()
75		self.config = config.read_config(
76		os.path.dirname(__file__) + "/executor.cfg",
77		override_url=self.live_server_url)
78
79		def _register_executor(self):
80		server.send_hostinfo(self.config)
81		return TestMachine.objects.order_by('-last_contact')[0]
82
83		def _run_executor(self):
84		return cmdline.download_and_run(self.config)
85
86		def test_send_result(self):
87		sf = create_submission_file()
88		sub = create_validatable_submission(
89		self.user, self.validated_assignment, sf)
90		test_machine = self._register_executor()
91		sub.assignment.test_machines.add(test_machine)
92
93		job = server.fetch_job(self.config)
94		self.assertNotEquals(None, job)
95
96		msg = uccrap + "Message from validation script."
97
98		job.send_pass_result(info_student=msg)
99
100		db_entries = SubmissionTestResult.objects.filter(
101		kind=SubmissionTestResult.VALIDITY_TEST)
102
103		self.assertEqual(db_entries[0].result, msg)
104
105		def test_fetch_job_renaming(self):
106		test_machine = self._register_executor()
107		self.validated_assignment.test_machines.add(test_machine)
108		with open(rootdir + "submfiles/validation/1000fff/helloworld.c", 'rb') as f:
109		self.c.post('/assignments/%s/new/' % self.validated_assignment.pk, {
110		'notes': 'This is a test submission.',
111		'authors': str(self.user.pk),
112		'attachment': f
113		})
114		job = server.fetch_job(self.config)
115		self.assertIn('helloworld.c', os.listdir(job.working_dir))
116
117		def test_job_attributes(self):
118		sf = create_submission_file()
119		sub = create_validatable_submission(
120		self.user, self.validated_assignment, sf)
121		test_machine = self._register_executor()
122		sub.assignment.test_machines.add(test_machine)
123		job = server.fetch_job(self.config)
124		self.assertNotEquals(None, job)
125		self.assertEquals(job.timeout, self.validated_assignment.attachment_test_timeout)
126		self.assertEquals(job.sub_id, str(sub.pk))
127		self.assertEquals(job.file_id, str(sf.pk))
128		self.assertEquals(job.submitter_name, sub.submitter.get_full_name())
129		self.assertEquals(job.submitter_student_id, str(sub.submitter.profile.student_id))
130		self.assertEquals(job.submitter_studyprogram, str(sub.submitter.profile.study_program))
131		self.assertEquals(job.course, str(self.course))
132		self.assertEquals(job.assignment, str(self.validated_assignment))
133
134		def test_wrong_compile_call(self):
135		sf = create_submission_file()
136		sub = create_validatable_submission(
137		self.user, self.validated_assignment, sf)
138		test_machine = self._register_executor()
139		sub.assignment.test_machines.add(test_machine)
140		job = server.fetch_job(self.config)
141		# Missing output
142		self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, inputs=['input.c'])
143		# Missing input
144		self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, output='output')
145		# Wrong file, determined by compiler
146		self.assertRaises(exceptions.WrongExitStatusException, job.run_compiler, inputs=['foo.c'], output='output')
147
148
149		class Validation(TestCase):
150		'''
151		Tests for the execution of validation scripts by the executor.
152		'''
153
154		def setUp(self):
155		super(Validation, self).setUp()
156		self.config = config.read_config(
157		os.path.dirname(__file__) + "/executor.cfg")
158
159		def _test_validation_case(self, directory):
160		'''
161		Each of the validator.py files uses the Python assert()
162		statement to check by itself if the result is the expected
163		one for its case.
164		'''
165		base_dir = os.path.dirname(__file__) + '/submfiles/validation/'
166		cmdline.copy_and_run(self.config, base_dir + directory)
167
168		def test_0100fff(self):
169		self._test_validation_case('0100fff')
170
171		def test_0100tff(self):
172		self._test_validation_case('0100tff')
173
174		def test_0100ttf(self):
175		self._test_validation_case('0100ttf')
176
177		def test_1000fff(self):
178		self._test_validation_case('1000fff')
179
180		def test_1000fft(self):
181		self._test_validation_case('1000fft')
182
183		def test_1000tff(self):
184		self._test_validation_case('1000tff')
185
186		def test_1000tft(self):
187		self._test_validation_case('1000tft')
188
189		def test_1000ttf(self):
190		self._test_validation_case('1000ttf')
191
192		def test_1000ttt(self):
193		self._test_validation_case('1000ttt')
194
195		def test_1010tff(self):
196		self._test_validation_case('1010tff')
197
198		def test_1010ttf(self):
199		self._test_validation_case('1010ttf')
200
201		def test_1100tff(self):
202		self._test_validation_case('1100tff')
203
204		def test_1100ttf(self):
205		self._test_validation_case('1100ttf')
206
207		def test_3000tff(self):
208		self._test_validation_case('3000tff')
209
210		def test_3000ttf(self):
211		self._test_validation_case('3000ttf')
212
213		def test_3010tff(self):
214		self._test_validation_case('3010tff')
215
216		def test_3010ttf(self):
217		self._test_validation_case('3010ttf')
218
219		def test_b000tff(self):
220		self._test_validation_case('b000tff')
221
222		def test_b010tff(self):
223		self._test_validation_case('b010tff')
224
225		def test_1000tfm(self):
226		self._test_validation_case('1000tfm')
227
228
229		class Communication(SubmitStudentScenarioTestCase):
230		'''
231		Tests for the communication of the executor with the OpenSubmit server.
232		'''
233
234		def setUp(self):
235		settings.MAIN_URL = self.live_server_url
236		super(Communication, self).setUp()
237		self.config = config.read_config(
238		os.path.dirname(__file__) + "/executor.cfg",
239		override_url=self.live_server_url)
240
241		def _register_executor(self):
242		server.send_hostinfo(self.config)
243		return TestMachine.objects.order_by('-last_contact')[0]
244
245		def _run_executor(self):
246		return cmdline.download_and_run(self.config)
247
248		def _register_test_machine(self):
249		'''
250		Utility step for a common test case preparation:
251		- Create validatable submission
252		- Register a test machine for it
253		'''
254		sf = create_submission_file()
255		sub = create_validatable_submission(
256		self.user, self.validated_assignment, sf)
257		test_machine = self._register_executor()
258		sub.assignment.test_machines.add(test_machine)
259		return sub
260
261		def test_register_executor_explicit(self):
262		machine_count = TestMachine.objects.all().count()
263		assert(self._register_executor().pk)
264		self.assertEqual(machine_count + 1, TestMachine.objects.all().count())
265
266		@override_settings(JOB_EXECUTOR_SECRET='foo')
267		def test_invalid_secret(self):
268		self.assertNotEqual(True, self._run_executor())
269
270		def test_everything_already_tested(self):
271		create_validated_submission(self.user, self.validated_assignment)
272		assert(self._register_executor().pk)
273		self.assertEqual(False, self._run_executor())
274
275		def test_parallel_executors_test(self):
276		NUM_PARALLEL = 3
277		self.validated_assignment.test_machines.add(self._register_executor())
278		subs = []
279		for i in range(1, NUM_PARALLEL + 1):
280		stud = create_user(get_student_dict(i))
281		self.course.participants.add(stud.profile)
282		self.course.save()
283		sf = create_submission_file()
284		subs.append(create_validatable_submission(
285		stud, self.validated_assignment, sf))
286
287		# Span a number of threads, each triggering the executor
288		# This only creates a real test case if executor serialization
289		# is off (see tests/executor.cfg)
290		return_codes = utils.run_parallel(len(subs), self._run_executor)
291		self.assertEqual(
292		len(list(filter((lambda x: x is True), return_codes))),
293		len(subs))
294
295		for sub in subs:
296		results = SubmissionTestResult.objects.filter(
297		kind=SubmissionTestResult.VALIDITY_TEST
298		)
299		self.assertEqual(NUM_PARALLEL, len(results))
300		self.assertNotEqual(0, len(results[0].result))
301
302	View Code Duplication	def test_too_long_validation(self):
		0 ignored issues – show Duplication introduced 2017-11-30 10:01 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
303		from django.core import mail
304
305		grading = create_pass_fail_grading()
306		assignment = create_validated_assignment(
307		self.course, grading, "/submfiles/validation/d000fff/", "validator_run.py")
308		assignment.attachment_test_timeout = 1
309		assignment.save()
310		sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
311		sub = create_validatable_submission(
312		self.user, assignment, sf)
313		test_machine = self._register_executor()
314		sub.assignment.test_machines.add(test_machine)
315
316		# Fire up the executor, should mark the submission as timed out
317		self.assertEqual(True, self._run_executor())
318		# Check if timeout marking took place
319		sub.refresh_from_db()
320		self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
321		text = sub.get_validation_result().result
322		self.assertIn("took too long", text)
323		# Check mail outbox for student information
324		self.assertEqual(1, len(mail.outbox))
325		for email in mail.outbox:
326		self.assertIn("Validation failed", email.subject)
327		self.assertIn("failed", email.body)
328		self.assertIn("localhost", email.body)
329
330	View Code Duplication	def test_broken_validator_feedback(self):
		0 ignored issues – show Duplication introduced 2017-12-16 11:35 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
331		from django.core import mail
332
333		grading = create_pass_fail_grading()
334		assignment = create_validated_assignment(
335		self.course, grading, "/submfiles/validation/1000tfm/", "validator.zip")
336		assignment.save()
337		sf = create_submission_file("/submfiles/validation/1000tfm/packed.zip")
338		sub = create_validatable_submission(
339		self.user, assignment, sf)
340		test_machine = self._register_executor()
341		sub.assignment.test_machines.add(test_machine)
342
343		# Fire up the executor
344		self.assertEqual(False, self._run_executor())
345		sub.refresh_from_db()
346		self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
347		text = sub.get_validation_result().result
348		self.assertIn("Internal error", text)
349		# Check mail outbox for student information
350		self.assertEqual(1, len(mail.outbox))
351		for email in mail.outbox:
352		self.assertIn("Validation failed", email.subject)
353		self.assertIn("failed", email.body)
354		self.assertIn("localhost", email.body)
355
356	View Code Duplication	def test_too_long_full_test(self):
		0 ignored issues – show Duplication introduced 2017-11-30 10:01 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
357		grading = create_pass_fail_grading()
358		assignment = create_validated_assignment(
359		self.course,
360		grading,
361		"/submfiles/validation/d000fff/",
362		"validator_build.py",
363		"validator_run.py")
364		assignment.attachment_test_timeout = 1
365		assignment.save()
366		sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
367		sub = create_validatable_submission(
368		self.user, assignment, sf)
369		test_machine = self._register_executor()
370		sub.assignment.test_machines.add(test_machine)
371
372		# Fire up the executor for validation
373		self.assertEqual(True, self._run_executor())
374		# Fire up the executor for full test
375		self.assertEqual(True, self._run_executor())
376		# Check if timeout marking took place
377		sub.refresh_from_db()
378		self.assertEqual(sub.state, Submission.TEST_FULL_FAILED)
379		assert("took too long" in sub.get_fulltest_result().result)
380		# Failed full tests shall not be reported
381		self.assertEqual(0, len(mail.outbox))
382
383		def test_single_file_validator_test(self):
384		sub = self._register_test_machine()
385
386		self.assertEqual(True, self._run_executor())
387		results = SubmissionTestResult.objects.filter(
388		submission_file=sub.file_upload,
389		kind=SubmissionTestResult.VALIDITY_TEST
390		)
391		self.assertEqual(1, len(results))
392
393		self.assertEqual(True, self._run_executor())
394		results = SubmissionTestResult.objects.filter(
395		submission_file=sub.file_upload,
396		kind=SubmissionTestResult.VALIDITY_TEST
397		)
398		self.assertEqual(1, len(results))
399		self.assertNotEqual(0, len(results[0].result))
400		# Graded assignment, so no mail at this stage
401		self.assertEqual(0, len(mail.outbox))
402
403	View Code Duplication	def test_ungraded_validation_email(self):
		0 ignored issues – show Duplication introduced 2017-12-16 11:35 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
404		from django.core import mail
405
406		sf = create_submission_file()
407		assign = create_validated_assignment_with_file(self.course, None)
408		sub = create_validatable_submission(self.user, assign, sf)
409		test_machine = self._register_executor()
410		assign.test_machines.add(test_machine)
411		# has no grading scheme, so it is not graded
412		assert(not sub.assignment.is_graded())
413
414		# Fire up the executor for validation
415		self.assertEqual(True, self._run_executor())
416
417		# Fire up the executor for full test
418		self.assertEqual(True, self._run_executor())
419
420		# Check mail outbox for student information
421		self.assertEqual(1, len(mail.outbox))
422		for email in mail.outbox:
423		self.assertIn("Validation successful", email.subject)
424		self.assertIn("passed", email.body)
425		self.assertIn("localhost", email.body)
426
427		def test_full_test(self):
428		sub = self._register_test_machine()
429		# validation test
430		self.assertEqual(True, self._run_executor())
431		# full test
432		self.assertEqual(True, self._run_executor())
433		results = SubmissionTestResult.objects.filter(
434		submission_file=sub.file_upload,
435		kind=SubmissionTestResult.FULL_TEST
436		)
437		self.assertEqual(1, len(results))
438		self.assertNotEqual(0, len(results[0].result))
439
440		def test_inconsistent_state_email(self):
441		'''
442		Test operator email on inconsistent state.
443		Since executor execution and submission sending is one step,
444		we need to mock the incoming invalid executor request.
445		'''
446		sf = create_submission_file()
447		self.sub = create_validatable_submission(
448		self.user, self.validated_assignment, sf)
449		test_machine = self._register_executor()
450		self.sub.assignment.test_machines.add(test_machine)
451		self.sub.state = Submission.TEST_FULL_PENDING
452		self.sub.save()
453		post_data = {'Secret': settings.JOB_EXECUTOR_SECRET,
454		'UUID': test_machine.host,
455		'Action': 'test_compile',
456		'SubmissionFileId': self.sub.file_upload.pk,
457		'PerfData': '',
458		'ErrorCode': 0,
459		'Message': 'In A Bottle'}
460		self.c.post(reverse('jobs'), post_data)
461		self.assertNotEqual(len(mail.outbox), 0)
462		self.assertIn('Action reported by the executor: test_compile',
463		mail.outbox[0].message().as_string())
464
465		def test_assignment_specific_test_machine(self):
466		# Register two test machines T1 and T2
467		real_machine = self._register_executor()
468		fake_machine = TestMachine(host="127.0.0.2")
469		fake_machine.save()
470		# Assign each of them to a different assignment
471		self.open_assignment.test_machines.add(real_machine)
472		self.validated_assignment.test_machines.add(fake_machine)
473		# Produce submission for the assignment linked to fake_machine
474		sub1 = Submission(
475		assignment=self.validated_assignment,
476		submitter=self.user,
477		state=Submission.TEST_VALIDITY_PENDING,
478		file_upload=create_submission_file()
479		)
480		sub1.save()
481		# Run real_machine executor, should not react on this submission
482		old_sub1_state = sub1.state
483		self.assertEqual(False, self._run_executor())
484		# Make sure that submission object was not touched,
485		# whatever the executor says
486		sub1 = Submission.objects.get(pk=sub1.pk)
487		self.assertEqual(old_sub1_state, sub1.state)
488

troeger / opensubmit

Push — master ( 7d61bb...58af22 )

Communication.test_too_long_validation() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like