Library.test_cleanup() - Code Metrics - Inspection of "Add missing tests" - troeger/opensubmit - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 03c658...45831a )

by Peter

created 2018-01-08 18:11 UTC

Library.test_cleanup() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	1

Metric	Value
c	1
b	0
f	1
dl	0
loc	21
rs	9.0534
cc	4

'''
    Test cases for the executor code.

    Note the neccessary python search path manipulation below.

Missing tests:

  - Makefile in validator package should override student Makefile

'''

import os
import os.path
import sys
import logging

from django.core import mail
from django.conf import settings
from django.test import TestCase
from django.test.utils import override_settings
from opensubmit.tests.cases import SubmitStudentScenarioTestCase
from django.core.urlresolvers import reverse

from opensubmit.models import TestMachine, SubmissionTestResult, Submission
from opensubmit.tests import utils

from .helpers.submission import create_validatable_submission
from .helpers.submission import create_validated_submission
from .helpers.assignment import create_validated_assignment
from .helpers.assignment import create_pass_fail_grading
from .helpers.assignment import create_validated_assignment_with_file
from .helpers.djangofiles import create_submission_file
from .helpers.user import create_user, get_student_dict
from . import uccrap, rootdir

sys.path.insert(0, os.path.dirname(__file__) + '/../../../executor/')
from opensubmitexec import config, cmdline, server, locking, compiler, exceptions  # NOQA

logger = logging.getLogger('opensubmitexec')


class CmdLine(TestCase):
    '''
    Test cases for the executor command-line script.
    '''
    def setUp(self):
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg")

    def test_help_call_without_config(self):
        # simulate 'help' call
        sys.argv = ['opensubmit-exec', 'help']
        cmdline.console_script()

    def test_unlock_call_without_lock(self):
        # simulate 'unlock' call without existing lock
        sys.argv = ['opensubmit-exec', 'unlock']
        cmdline.console_script()

    def test_unlock_call_with_lock(self):
        # simulate 'unlock' call with existing lock
        sys.argv = ['opensubmit-exec', 'unlock']
        with locking.ScriptLock(self.config):
            cmdline.console_script()


class Library(SubmitStudentScenarioTestCase):
    '''
    Tests for the executor library functions used by the validator script.
    '''
    def setUp(self):
        settings.MAIN_URL = self.live_server_url
        super(Library, self).setUp()
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg",
            override_url=self.live_server_url)

    def _register_executor(self):
        server.send_hostinfo(self.config)
        return TestMachine.objects.order_by('-last_contact')[0]

    def _run_executor(self):
        return cmdline.download_and_run(self.config)

    def test_send_result(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)

        msg = uccrap + "Message from validation script."

        job.send_pass_result(info_student=msg)

        db_entries = SubmissionTestResult.objects.filter(
            kind=SubmissionTestResult.VALIDITY_TEST)

        self.assertEqual(db_entries[0].result, msg)

    def test_fetch_job_renaming(self):
        test_machine = self._register_executor()
        self.validated_assignment.test_machines.add(test_machine)
        with open(rootdir + "submfiles/validation/1000fff/helloworld.c", 'rb') as f:
            self.c.post('/assignments/%s/new/' % self.validated_assignment.pk, {
                'notes': 'This is a test submission.',
                'authors': str(self.user.pk),
                'attachment': f
            })
        job = server.fetch_job(self.config)
        self.assertIn('helloworld.c', os.listdir(job.working_dir))

    def test_grep(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)
        self.assertListEqual(job.grep("World"), ['helloworld.c'])
        self.assertListEqual(job.grep("foobar"), [])

    def test_ensure_files(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)
        self.assertTrue(job.ensure_files(['helloworld.c']))

    def test_job_attributes(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)
        self.assertEquals(job.timeout, self.validated_assignment.attachment_test_timeout)
        self.assertEquals(job.sub_id, str(sub.pk))
        self.assertEquals(job.file_id, str(sf.pk))
        self.assertEquals(job.submitter_name, sub.submitter.get_full_name())
        self.assertEquals(job.submitter_student_id, str(sub.submitter.profile.student_id))
        self.assertEquals(job.submitter_studyprogram, str(sub.submitter.profile.study_program))
        self.assertEquals(job.course, str(self.course))
        self.assertEquals(job.assignment, str(self.validated_assignment))

    def test_wrong_compile_call(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        # Missing output
        self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, inputs=['input.c'])
        # Missing input
        self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, output='output')
        # Wrong file, determined by compiler
        self.assertRaises(exceptions.WrongExitStatusException, job.run_compiler, inputs=['foo.c'], output='output')

    def test_cleanup(self):
        '''
        If configured, then the executor should remove all temporary
        files after finishing.
        '''
        for do_cleanup in [False, True]:
            sf = create_submission_file()
            sub = create_validatable_submission(
                self.user, self.validated_assignment, sf)
            test_machine = self._register_executor()
            sub.assignment.test_machines.add(test_machine)
            job = server.fetch_job(self.config)
            self.assertNotEquals(None, job)
            working_dir = job.working_dir
            job._config.set("Execution", "cleanup", str(do_cleanup))
            job._run_validate()
            if do_cleanup:
                with self.assertRaises(FileNotFoundError):
                    os.listdir(working_dir)
            else:
                self.assertNotEqual(os.listdir(working_dir), [])



class Validation(TestCase):
    '''
    Tests for the execution of validation scripts by the executor.
    '''

    def setUp(self):
        super(Validation, self).setUp()
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg")

    def _test_validation_case(self, directory):
        '''
        Each of the validator.py files uses the Python assert()
        statement to check by itself if the result is the expected
        one for its case.
        '''
        base_dir = os.path.dirname(__file__) + '/submfiles/validation/'
        cmdline.copy_and_run(self.config, base_dir + directory)

    def test_0100fff(self):
        self._test_validation_case('0100fff')

    def test_0100tff(self):
        self._test_validation_case('0100tff')

    def test_0100ttf(self):
        self._test_validation_case('0100ttf')

    def test_1000fff(self):
        self._test_validation_case('1000fff')

    def test_1000fft(self):
        self._test_validation_case('1000fft')

    def test_1000tff(self):
        self._test_validation_case('1000tff')

    def test_1000tft(self):
        self._test_validation_case('1000tft')

    def test_1000ttf(self):
        self._test_validation_case('1000ttf')

    def test_1000ttt(self):
        self._test_validation_case('1000ttt')

    def test_1010tff(self):
        self._test_validation_case('1010tff')

    def test_1010ttf(self):
        self._test_validation_case('1010ttf')

    def test_1100tff(self):
        self._test_validation_case('1100tff')

    def test_1100ttf(self):
        self._test_validation_case('1100ttf')

    def test_3000tff(self):
        self._test_validation_case('3000tff')

    def test_3000ttf(self):
        self._test_validation_case('3000ttf')

    def test_3010tff(self):
        self._test_validation_case('3010tff')

    def test_3010ttf(self):
        self._test_validation_case('3010ttf')

    def test_b000tff(self):
        self._test_validation_case('b000tff')

    def test_b010tff(self):
        self._test_validation_case('b010tff')

    def test_1000tfm(self):
        self._test_validation_case('1000tfm')


class Communication(SubmitStudentScenarioTestCase):
    '''
    Tests for the communication of the executor with the OpenSubmit server.
    '''

    def setUp(self):
        settings.MAIN_URL = self.live_server_url
        super(Communication, self).setUp()
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg",
            override_url=self.live_server_url)

    def _register_executor(self):
        server.send_hostinfo(self.config)
        return TestMachine.objects.order_by('-last_contact')[0]

    def _run_executor(self):
        return cmdline.download_and_run(self.config)

    def _register_test_machine(self):
        '''
        Utility step for a common test case preparation:
        - Create validatable submission
        - Register a test machine for it
        '''
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        return sub


    def test_register_executor_explicit(self):
        machine_count = TestMachine.objects.all().count()
        assert(self._register_executor().pk)
        self.assertEqual(machine_count + 1, TestMachine.objects.all().count())

    @override_settings(JOB_EXECUTOR_SECRET='foo')
    def test_invalid_secret(self):
        self.assertNotEqual(True, self._run_executor())

    def test_everything_already_tested(self):
        create_validated_submission(self.user, self.validated_assignment)
        assert(self._register_executor().pk)
        self.assertEqual(False, self._run_executor())

    def test_parallel_executors_test(self):
        NUM_PARALLEL = 3
        self.validated_assignment.test_machines.add(self._register_executor())
        subs = []
        for i in range(1, NUM_PARALLEL + 1):
            stud = create_user(get_student_dict(i))
            self.course.participants.add(stud.profile)
            self.course.save()
            sf = create_submission_file()
            subs.append(create_validatable_submission(
                stud, self.validated_assignment, sf))

        # Span a number of threads, each triggering the executor

        # This only creates a real test case if executor serialization
        # is off (see tests/executor.cfg)
        return_codes = utils.run_parallel(len(subs), self._run_executor)
        self.assertEqual(
            len(list(filter((lambda x: x is True), return_codes))),
            len(subs))

        for sub in subs:
            results = SubmissionTestResult.objects.filter(
                kind=SubmissionTestResult.VALIDITY_TEST
            )
            self.assertEqual(NUM_PARALLEL, len(results))
            self.assertNotEqual(0, len(results[0].result))

    def test_too_long_validation(self):
        from django.core import mail

        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, grading, "/submfiles/validation/d000fff/", "validator_run.py")
        assignment.attachment_test_timeout = 1
        assignment.save()
        sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()

        sub.assignment.test_machines.add(test_machine)

        # Fire up the executor, should mark the submission as timed out
        self.assertEqual(True, self._run_executor())
        # Check if timeout marking took place
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
        text = sub.get_validation_result().result
        self.assertIn("took too long", text)
        # Check mail outbox for student information
        self.assertEqual(1, len(mail.outbox))
        for email in mail.outbox:
            self.assertIn("Validation failed", email.subject)
            self.assertIn("failed", email.body)
            self.assertIn("localhost", email.body)

    def test_broken_validator_feedback(self):
        from django.core import mail


        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, grading, "/submfiles/validation/1000tfm/", "validator.zip")
        assignment.save()
        sf = create_submission_file("/submfiles/validation/1000tfm/packed.zip")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        # Fire up the executor
        self.assertEqual(False, self._run_executor())
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
        text = sub.get_validation_result().result
        self.assertIn("Internal error", text)
        # Check mail outbox for student information
        self.assertEqual(1, len(mail.outbox))
        for email in mail.outbox:
            self.assertIn("Validation failed", email.subject)
            self.assertIn("failed", email.body)
            self.assertIn("localhost", email.body)

    def test_output_logging(self):
        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, grading, "/submfiles/validation/1000fff/", "validator.py")
        assignment.save()
        sf = create_submission_file("/submfiles/validation/1000fff/helloworld.c")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        # Fire up the executor
        self.assertEqual(True, self._run_executor())
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.SUBMITTED_TESTED)
        text = sub.get_validation_result().result
        self.assertIn("quick brown fox", text)
        self.assertIn("provide your input", text)

    def test_too_long_full_test(self):
        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, 
            grading, 

            "/submfiles/validation/d000fff/",
            "validator_build.py",
            "validator_run.py")
        assignment.attachment_test_timeout = 1
        assignment.save()
        sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        # Fire up the executor for validation
        self.assertEqual(True, self._run_executor())
        # Fire up the executor for full test
        self.assertEqual(True, self._run_executor())
        # Check if timeout marking took place
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.TEST_FULL_FAILED)
        assert("took too long" in sub.get_fulltest_result().result)
        # Failed full tests shall not be reported
        self.assertEqual(0, len(mail.outbox))

    def test_single_file_validator_test(self):
        sub = self._register_test_machine()

        self.assertEqual(True, self._run_executor())
        results = SubmissionTestResult.objects.filter(
            submission_file=sub.file_upload,
            kind=SubmissionTestResult.VALIDITY_TEST
        )
        self.assertEqual(1, len(results))

        self.assertEqual(True, self._run_executor())
        results = SubmissionTestResult.objects.filter(
            submission_file=sub.file_upload,
            kind=SubmissionTestResult.VALIDITY_TEST
        )
        self.assertEqual(1, len(results))
        self.assertNotEqual(0, len(results[0].result))
        # Graded assignment, so no mail at this stage
        self.assertEqual(0, len(mail.outbox))

    def test_ungraded_validation_email(self):
        from django.core import mail

        sf = create_submission_file()
        assign = create_validated_assignment_with_file(self.course, None)
        sub = create_validatable_submission(self.user, assign, sf)
        test_machine = self._register_executor()
        assign.test_machines.add(test_machine)
        # has no grading scheme, so it is not graded
        assert(not sub.assignment.is_graded())

        # Fire up the executor for validation
        self.assertEqual(True, self._run_executor())

        # Fire up the executor for full test
        self.assertEqual(True, self._run_executor())

        # Check mail outbox for student information
        self.assertEqual(1, len(mail.outbox))
        for email in mail.outbox:
            self.assertIn("Validation successful", email.subject)
            self.assertIn("passed", email.body)
            self.assertIn("localhost", email.body)

    def test_full_test(self):
        sub = self._register_test_machine()
        # validation test
        self.assertEqual(True, self._run_executor())
        # full test
        self.assertEqual(True, self._run_executor())
        results = SubmissionTestResult.objects.filter(
            submission_file=sub.file_upload,
            kind=SubmissionTestResult.FULL_TEST
        )
        self.assertEqual(1, len(results))
        self.assertNotEqual(0, len(results[0].result))

    def test_inconsistent_state_email(self):
        '''
        Test operator email on inconsistent state.
        Since executor execution and submission sending is one step,
        we need to mock the incoming invalid executor request.
        '''
        sf = create_submission_file()
        self.sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        self.sub.assignment.test_machines.add(test_machine)
        self.sub.state = Submission.TEST_FULL_PENDING
        self.sub.save()
        post_data = {'Secret': settings.JOB_EXECUTOR_SECRET,
                     'UUID': test_machine.host,
                     'Action': 'test_compile',
                     'SubmissionFileId': self.sub.file_upload.pk,
                     'PerfData': '',
                     'ErrorCode': 0,
                     'Message': 'In A Bottle'}
        self.c.post(reverse('jobs'), post_data)
        self.assertNotEqual(len(mail.outbox), 0)
        self.assertIn('Action reported by the executor: test_compile',
                      mail.outbox[0].message().as_string())

    def test_assignment_specific_test_machine(self):
        # Register two test machines T1 and T2
        real_machine = self._register_executor()
        fake_machine = TestMachine(host="127.0.0.2")
        fake_machine.save()
        # Assign each of them to a different assignment
        self.open_assignment.test_machines.add(real_machine)
        self.validated_assignment.test_machines.add(fake_machine)
        # Produce submission for the assignment linked to fake_machine
        sub1 = Submission(
            assignment=self.validated_assignment,
            submitter=self.user,
            state=Submission.TEST_VALIDITY_PENDING,
            file_upload=create_submission_file()
        )
        sub1.save()
        # Run real_machine executor, should not react on this submission
        old_sub1_state = sub1.state
        self.assertEqual(False, self._run_executor())
        # Make sure that submission object was not touched,
        # whatever the executor says
        sub1 = Submission.objects.get(pk=sub1.pk)
        self.assertEqual(old_sub1_state, sub1.state)


1		'''
2		Test cases for the executor code.
3
4		Note the neccessary python search path manipulation below.
5
6		Missing tests:
7
8		- Makefile in validator package should override student Makefile
9
10		'''
11
12		import os
13		import os.path
14		import sys
15		import logging
16
17		from django.core import mail
18		from django.conf import settings
19		from django.test import TestCase
20		from django.test.utils import override_settings
21		from opensubmit.tests.cases import SubmitStudentScenarioTestCase
22		from django.core.urlresolvers import reverse
23
24		from opensubmit.models import TestMachine, SubmissionTestResult, Submission
25		from opensubmit.tests import utils
26
27		from .helpers.submission import create_validatable_submission
28		from .helpers.submission import create_validated_submission
29		from .helpers.assignment import create_validated_assignment
30		from .helpers.assignment import create_pass_fail_grading
31		from .helpers.assignment import create_validated_assignment_with_file
32		from .helpers.djangofiles import create_submission_file
33		from .helpers.user import create_user, get_student_dict
34		from . import uccrap, rootdir
35
36		sys.path.insert(0, os.path.dirname(__file__) + '/../../../executor/')
37		from opensubmitexec import config, cmdline, server, locking, compiler, exceptions # NOQA
38
39		logger = logging.getLogger('opensubmitexec')
40
41
42		class CmdLine(TestCase):
43		'''
44		Test cases for the executor command-line script.
45		'''
46		def setUp(self):
47		self.config = config.read_config(
48		os.path.dirname(__file__) + "/executor.cfg")
49
50		def test_help_call_without_config(self):
51		# simulate 'help' call
52		sys.argv = ['opensubmit-exec', 'help']
53		cmdline.console_script()
54
55		def test_unlock_call_without_lock(self):
56		# simulate 'unlock' call without existing lock
57		sys.argv = ['opensubmit-exec', 'unlock']
58		cmdline.console_script()
59
60		def test_unlock_call_with_lock(self):
61		# simulate 'unlock' call with existing lock
62		sys.argv = ['opensubmit-exec', 'unlock']
63		with locking.ScriptLock(self.config):
64		cmdline.console_script()
65
66
67		class Library(SubmitStudentScenarioTestCase):
68		'''
69		Tests for the executor library functions used by the validator script.
70		'''
71		def setUp(self):
72		settings.MAIN_URL = self.live_server_url
73		super(Library, self).setUp()
74		self.config = config.read_config(
75		os.path.dirname(__file__) + "/executor.cfg",
76		override_url=self.live_server_url)
77
78		def _register_executor(self):
79		server.send_hostinfo(self.config)
80		return TestMachine.objects.order_by('-last_contact')[0]
81
82		def _run_executor(self):
83		return cmdline.download_and_run(self.config)
84
85		def test_send_result(self):
86		sf = create_submission_file()
87		sub = create_validatable_submission(
88		self.user, self.validated_assignment, sf)
89		test_machine = self._register_executor()
90		sub.assignment.test_machines.add(test_machine)
91
92		job = server.fetch_job(self.config)
93		self.assertNotEquals(None, job)
94
95		msg = uccrap + "Message from validation script."
96
97		job.send_pass_result(info_student=msg)
98
99		db_entries = SubmissionTestResult.objects.filter(
100		kind=SubmissionTestResult.VALIDITY_TEST)
101
102		self.assertEqual(db_entries[0].result, msg)
103
104		def test_fetch_job_renaming(self):
105		test_machine = self._register_executor()
106		self.validated_assignment.test_machines.add(test_machine)
107		with open(rootdir + "submfiles/validation/1000fff/helloworld.c", 'rb') as f:
108		self.c.post('/assignments/%s/new/' % self.validated_assignment.pk, {
109		'notes': 'This is a test submission.',
110		'authors': str(self.user.pk),
111		'attachment': f
112		})
113		job = server.fetch_job(self.config)
114		self.assertIn('helloworld.c', os.listdir(job.working_dir))
115
116		def test_grep(self):
117		sf = create_submission_file()
118		sub = create_validatable_submission(
119		self.user, self.validated_assignment, sf)
120		test_machine = self._register_executor()
121		sub.assignment.test_machines.add(test_machine)
122		job = server.fetch_job(self.config)
123		self.assertNotEquals(None, job)
124		self.assertListEqual(job.grep("World"), ['helloworld.c'])
125		self.assertListEqual(job.grep("foobar"), [])
126
127		def test_ensure_files(self):
128		sf = create_submission_file()
129		sub = create_validatable_submission(
130		self.user, self.validated_assignment, sf)
131		test_machine = self._register_executor()
132		sub.assignment.test_machines.add(test_machine)
133		job = server.fetch_job(self.config)
134		self.assertNotEquals(None, job)
135		self.assertTrue(job.ensure_files(['helloworld.c']))
136
137		def test_job_attributes(self):
138		sf = create_submission_file()
139		sub = create_validatable_submission(
140		self.user, self.validated_assignment, sf)
141		test_machine = self._register_executor()
142		sub.assignment.test_machines.add(test_machine)
143		job = server.fetch_job(self.config)
144		self.assertNotEquals(None, job)
145		self.assertEquals(job.timeout, self.validated_assignment.attachment_test_timeout)
146		self.assertEquals(job.sub_id, str(sub.pk))
147		self.assertEquals(job.file_id, str(sf.pk))
148		self.assertEquals(job.submitter_name, sub.submitter.get_full_name())
149		self.assertEquals(job.submitter_student_id, str(sub.submitter.profile.student_id))
150		self.assertEquals(job.submitter_studyprogram, str(sub.submitter.profile.study_program))
151		self.assertEquals(job.course, str(self.course))
152		self.assertEquals(job.assignment, str(self.validated_assignment))
153
154		def test_wrong_compile_call(self):
155		sf = create_submission_file()
156		sub = create_validatable_submission(
157		self.user, self.validated_assignment, sf)
158		test_machine = self._register_executor()
159		sub.assignment.test_machines.add(test_machine)
160		job = server.fetch_job(self.config)
161		# Missing output
162		self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, inputs=['input.c'])
163		# Missing input
164		self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, output='output')
165		# Wrong file, determined by compiler
166		self.assertRaises(exceptions.WrongExitStatusException, job.run_compiler, inputs=['foo.c'], output='output')
167
168		def test_cleanup(self):
169		'''
170		If configured, then the executor should remove all temporary
171		files after finishing.
172		'''
173		for do_cleanup in [False, True]:
174		sf = create_submission_file()
175		sub = create_validatable_submission(
176		self.user, self.validated_assignment, sf)
177		test_machine = self._register_executor()
178		sub.assignment.test_machines.add(test_machine)
179		job = server.fetch_job(self.config)
180		self.assertNotEquals(None, job)
181		working_dir = job.working_dir
182		job._config.set("Execution", "cleanup", str(do_cleanup))
183		job._run_validate()
184		if do_cleanup:
185		with self.assertRaises(FileNotFoundError):
186		os.listdir(working_dir)
187		else:
188		self.assertNotEqual(os.listdir(working_dir), [])
189
190
191
192		class Validation(TestCase):
193		'''
194		Tests for the execution of validation scripts by the executor.
195		'''
196
197		def setUp(self):
198		super(Validation, self).setUp()
199		self.config = config.read_config(
200		os.path.dirname(__file__) + "/executor.cfg")
201
202		def _test_validation_case(self, directory):
203		'''
204		Each of the validator.py files uses the Python assert()
205		statement to check by itself if the result is the expected
206		one for its case.
207		'''
208		base_dir = os.path.dirname(__file__) + '/submfiles/validation/'
209		cmdline.copy_and_run(self.config, base_dir + directory)
210
211		def test_0100fff(self):
212		self._test_validation_case('0100fff')
213
214		def test_0100tff(self):
215		self._test_validation_case('0100tff')
216
217		def test_0100ttf(self):
218		self._test_validation_case('0100ttf')
219
220		def test_1000fff(self):
221		self._test_validation_case('1000fff')
222
223		def test_1000fft(self):
224		self._test_validation_case('1000fft')
225
226		def test_1000tff(self):
227		self._test_validation_case('1000tff')
228
229		def test_1000tft(self):
230		self._test_validation_case('1000tft')
231
232		def test_1000ttf(self):
233		self._test_validation_case('1000ttf')
234
235		def test_1000ttt(self):
236		self._test_validation_case('1000ttt')
237
238		def test_1010tff(self):
239		self._test_validation_case('1010tff')
240
241		def test_1010ttf(self):
242		self._test_validation_case('1010ttf')
243
244		def test_1100tff(self):
245		self._test_validation_case('1100tff')
246
247		def test_1100ttf(self):
248		self._test_validation_case('1100ttf')
249
250		def test_3000tff(self):
251		self._test_validation_case('3000tff')
252
253		def test_3000ttf(self):
254		self._test_validation_case('3000ttf')
255
256		def test_3010tff(self):
257		self._test_validation_case('3010tff')
258
259		def test_3010ttf(self):
260		self._test_validation_case('3010ttf')
261
262		def test_b000tff(self):
263		self._test_validation_case('b000tff')
264
265		def test_b010tff(self):
266		self._test_validation_case('b010tff')
267
268		def test_1000tfm(self):
269		self._test_validation_case('1000tfm')
270
271
272		class Communication(SubmitStudentScenarioTestCase):
273		'''
274		Tests for the communication of the executor with the OpenSubmit server.
275		'''
276
277		def setUp(self):
278		settings.MAIN_URL = self.live_server_url
279		super(Communication, self).setUp()
280		self.config = config.read_config(
281		os.path.dirname(__file__) + "/executor.cfg",
282		override_url=self.live_server_url)
283
284		def _register_executor(self):
285		server.send_hostinfo(self.config)
286		return TestMachine.objects.order_by('-last_contact')[0]
287
288		def _run_executor(self):
289		return cmdline.download_and_run(self.config)
290
291		def _register_test_machine(self):
292		'''
293		Utility step for a common test case preparation:
294		- Create validatable submission
295		- Register a test machine for it
296		'''
297		sf = create_submission_file()
298		sub = create_validatable_submission(
299		self.user, self.validated_assignment, sf)
300		test_machine = self._register_executor()
301		sub.assignment.test_machines.add(test_machine)
302	View Code Duplication	return sub
		0 ignored issues – show Duplication introduced 2018-01-08 18:12 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
303
304		def test_register_executor_explicit(self):
305		machine_count = TestMachine.objects.all().count()
306		assert(self._register_executor().pk)
307		self.assertEqual(machine_count + 1, TestMachine.objects.all().count())
308
309		@override_settings(JOB_EXECUTOR_SECRET='foo')
310		def test_invalid_secret(self):
311		self.assertNotEqual(True, self._run_executor())
312
313		def test_everything_already_tested(self):
314		create_validated_submission(self.user, self.validated_assignment)
315		assert(self._register_executor().pk)
316		self.assertEqual(False, self._run_executor())
317
318		def test_parallel_executors_test(self):
319		NUM_PARALLEL = 3
320		self.validated_assignment.test_machines.add(self._register_executor())
321		subs = []
322		for i in range(1, NUM_PARALLEL + 1):
323		stud = create_user(get_student_dict(i))
324		self.course.participants.add(stud.profile)
325		self.course.save()
326		sf = create_submission_file()
327		subs.append(create_validatable_submission(
328		stud, self.validated_assignment, sf))
329
330	View Code Duplication	# Span a number of threads, each triggering the executor
		0 ignored issues – show Duplication introduced 2018-01-08 18:12 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
331		# This only creates a real test case if executor serialization
332		# is off (see tests/executor.cfg)
333		return_codes = utils.run_parallel(len(subs), self._run_executor)
334		self.assertEqual(
335		len(list(filter((lambda x: x is True), return_codes))),
336		len(subs))
337
338		for sub in subs:
339		results = SubmissionTestResult.objects.filter(
340		kind=SubmissionTestResult.VALIDITY_TEST
341		)
342		self.assertEqual(NUM_PARALLEL, len(results))
343		self.assertNotEqual(0, len(results[0].result))
344
345		def test_too_long_validation(self):
346		from django.core import mail
347
348		grading = create_pass_fail_grading()
349		assignment = create_validated_assignment(
350		self.course, grading, "/submfiles/validation/d000fff/", "validator_run.py")
351		assignment.attachment_test_timeout = 1
352		assignment.save()
353		sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
354		sub = create_validatable_submission(
355		self.user, assignment, sf)
356	View Code Duplication	test_machine = self._register_executor()
		0 ignored issues – show Duplication introduced 2018-01-08 18:12 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
357		sub.assignment.test_machines.add(test_machine)
358
359		# Fire up the executor, should mark the submission as timed out
360		self.assertEqual(True, self._run_executor())
361		# Check if timeout marking took place
362		sub.refresh_from_db()
363		self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
364		text = sub.get_validation_result().result
365		self.assertIn("took too long", text)
366		# Check mail outbox for student information
367		self.assertEqual(1, len(mail.outbox))
368		for email in mail.outbox:
369		self.assertIn("Validation failed", email.subject)
370		self.assertIn("failed", email.body)
371		self.assertIn("localhost", email.body)
372
373		def test_broken_validator_feedback(self):
374	View Code Duplication	from django.core import mail
		0 ignored issues – show Duplication introduced 2018-01-08 18:12 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
375
376		grading = create_pass_fail_grading()
377		assignment = create_validated_assignment(
378		self.course, grading, "/submfiles/validation/1000tfm/", "validator.zip")
379		assignment.save()
380		sf = create_submission_file("/submfiles/validation/1000tfm/packed.zip")
381		sub = create_validatable_submission(
382		self.user, assignment, sf)
383		test_machine = self._register_executor()
384		sub.assignment.test_machines.add(test_machine)
385
386		# Fire up the executor
387		self.assertEqual(False, self._run_executor())
388		sub.refresh_from_db()
389		self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
390		text = sub.get_validation_result().result
391		self.assertIn("Internal error", text)
392		# Check mail outbox for student information
393		self.assertEqual(1, len(mail.outbox))
394		for email in mail.outbox:
395		self.assertIn("Validation failed", email.subject)
396		self.assertIn("failed", email.body)
397		self.assertIn("localhost", email.body)
398
399		def test_output_logging(self):
400		grading = create_pass_fail_grading()
401		assignment = create_validated_assignment(
402		self.course, grading, "/submfiles/validation/1000fff/", "validator.py")
403		assignment.save()
404		sf = create_submission_file("/submfiles/validation/1000fff/helloworld.c")
405		sub = create_validatable_submission(
406		self.user, assignment, sf)
407		test_machine = self._register_executor()
408		sub.assignment.test_machines.add(test_machine)
409		# Fire up the executor
410		self.assertEqual(True, self._run_executor())
411		sub.refresh_from_db()
412		self.assertEqual(sub.state, Submission.SUBMITTED_TESTED)
413		text = sub.get_validation_result().result
414		self.assertIn("quick brown fox", text)
415		self.assertIn("provide your input", text)
416
417		def test_too_long_full_test(self):
418		grading = create_pass_fail_grading()
419		assignment = create_validated_assignment(
420		self.course,
421	View Code Duplication	grading,
		0 ignored issues – show Duplication introduced 2018-01-08 18:12 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
422		"/submfiles/validation/d000fff/",
423		"validator_build.py",
424		"validator_run.py")
425		assignment.attachment_test_timeout = 1
426		assignment.save()
427		sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
428		sub = create_validatable_submission(
429		self.user, assignment, sf)
430		test_machine = self._register_executor()
431		sub.assignment.test_machines.add(test_machine)
432
433		# Fire up the executor for validation
434		self.assertEqual(True, self._run_executor())
435		# Fire up the executor for full test
436		self.assertEqual(True, self._run_executor())
437		# Check if timeout marking took place
438		sub.refresh_from_db()
439		self.assertEqual(sub.state, Submission.TEST_FULL_FAILED)
440		assert("took too long" in sub.get_fulltest_result().result)
441		# Failed full tests shall not be reported
442		self.assertEqual(0, len(mail.outbox))
443
444		def test_single_file_validator_test(self):
445		sub = self._register_test_machine()
446
447		self.assertEqual(True, self._run_executor())
448		results = SubmissionTestResult.objects.filter(
449		submission_file=sub.file_upload,
450		kind=SubmissionTestResult.VALIDITY_TEST
451		)
452		self.assertEqual(1, len(results))
453
454		self.assertEqual(True, self._run_executor())
455		results = SubmissionTestResult.objects.filter(
456		submission_file=sub.file_upload,
457		kind=SubmissionTestResult.VALIDITY_TEST
458		)
459		self.assertEqual(1, len(results))
460		self.assertNotEqual(0, len(results[0].result))
461		# Graded assignment, so no mail at this stage
462		self.assertEqual(0, len(mail.outbox))
463
464		def test_ungraded_validation_email(self):
465		from django.core import mail
466
467		sf = create_submission_file()
468		assign = create_validated_assignment_with_file(self.course, None)
469		sub = create_validatable_submission(self.user, assign, sf)
470		test_machine = self._register_executor()
471		assign.test_machines.add(test_machine)
472		# has no grading scheme, so it is not graded
473		assert(not sub.assignment.is_graded())
474
475		# Fire up the executor for validation
476		self.assertEqual(True, self._run_executor())
477
478		# Fire up the executor for full test
479		self.assertEqual(True, self._run_executor())
480
481		# Check mail outbox for student information
482		self.assertEqual(1, len(mail.outbox))
483		for email in mail.outbox:
484		self.assertIn("Validation successful", email.subject)
485		self.assertIn("passed", email.body)
486		self.assertIn("localhost", email.body)
487
488		def test_full_test(self):
489		sub = self._register_test_machine()
490		# validation test
491		self.assertEqual(True, self._run_executor())
492		# full test
493		self.assertEqual(True, self._run_executor())
494		results = SubmissionTestResult.objects.filter(
495		submission_file=sub.file_upload,
496		kind=SubmissionTestResult.FULL_TEST
497		)
498		self.assertEqual(1, len(results))
499		self.assertNotEqual(0, len(results[0].result))
500
501		def test_inconsistent_state_email(self):
502		'''
503		Test operator email on inconsistent state.
504		Since executor execution and submission sending is one step,
505		we need to mock the incoming invalid executor request.
506		'''
507		sf = create_submission_file()
508		self.sub = create_validatable_submission(
509		self.user, self.validated_assignment, sf)
510		test_machine = self._register_executor()
511		self.sub.assignment.test_machines.add(test_machine)
512		self.sub.state = Submission.TEST_FULL_PENDING
513		self.sub.save()
514		post_data = {'Secret': settings.JOB_EXECUTOR_SECRET,
515		'UUID': test_machine.host,
516		'Action': 'test_compile',
517		'SubmissionFileId': self.sub.file_upload.pk,
518		'PerfData': '',
519		'ErrorCode': 0,
520		'Message': 'In A Bottle'}
521		self.c.post(reverse('jobs'), post_data)
522		self.assertNotEqual(len(mail.outbox), 0)
523		self.assertIn('Action reported by the executor: test_compile',
524		mail.outbox[0].message().as_string())
525
526		def test_assignment_specific_test_machine(self):
527		# Register two test machines T1 and T2
528		real_machine = self._register_executor()
529		fake_machine = TestMachine(host="127.0.0.2")
530		fake_machine.save()
531		# Assign each of them to a different assignment
532		self.open_assignment.test_machines.add(real_machine)
533		self.validated_assignment.test_machines.add(fake_machine)
534		# Produce submission for the assignment linked to fake_machine
535		sub1 = Submission(
536		assignment=self.validated_assignment,
537		submitter=self.user,
538		state=Submission.TEST_VALIDITY_PENDING,
539		file_upload=create_submission_file()
540		)
541		sub1.save()
542		# Run real_machine executor, should not react on this submission
543		old_sub1_state = sub1.state
544		self.assertEqual(False, self._run_executor())
545		# Make sure that submission object was not touched,
546		# whatever the executor says
547		sub1 = Submission.objects.get(pk=sub1.pk)
548		self.assertEqual(old_sub1_state, sub1.state)
549

troeger / opensubmit

Push — master ( 03c658...45831a )

Library.test_cleanup() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like