Communication

Complexity

Total Complexity

27

Size/Duplication

Total Lines	278
Duplicated Lines	42.45 %

Importance

Changes	1
Bugs	0	Features	0

17 Methods

Rating	Name	Duplication	Size	Complexity
A	_register_executor()	0	3	1
A	test_assignment_specific_test_machine()	0	23	1
A	_run_executor()	0	2	1
A	setUp()	0	6	1
A	test_full_test()	0	13	1
B	test_inconsistent_state_email()	0	24	1
A	test_single_file_validator_test()	0	19	1
A	test_output_logging()	17	17	1
A	test_register_executor_explicit()	0	4	2
A	test_invalid_secret()	0	3	1
B	test_parallel_executors_test()	0	26	4
B	test_too_long_full_test()	26	26	2
B	test_broken_validator_feedback()	25	25	2
B	test_too_long_validation()	27	27	2
A	test_ungraded_validation_email()	23	23	3
A	_register_test_machine()	0	12	1
A	test_everything_already_tested()	0	4	2

'''
    Test cases for the executor code.

    Note the neccessary python search path manipulation below.

Missing tests:

  - Makefile in validator package should override student Makefile

'''

import os
import os.path
import sys
import logging

from django.core import mail
from django.conf import settings
from django.test import TestCase
from django.test.utils import override_settings
from opensubmit.tests.cases import SubmitStudentScenarioTestCase
from django.core.urlresolvers import reverse

from opensubmit.models import TestMachine, SubmissionTestResult, Submission
from opensubmit.tests import utils

from .helpers.submission import create_validatable_submission
from .helpers.submission import create_validated_submission
from .helpers.assignment import create_validated_assignment
from .helpers.assignment import create_pass_fail_grading
from .helpers.assignment import create_validated_assignment_with_file
from .helpers.djangofiles import create_submission_file
from .helpers.user import create_user, get_student_dict
from . import uccrap, rootdir

sys.path.insert(0, os.path.dirname(__file__) + '/../../../executor/')
from opensubmitexec import config, cmdline, server, locking, compiler, exceptions  # NOQA

logger = logging.getLogger('opensubmitexec')


class CmdLine(TestCase):
    '''
    Test cases for the executor command-line script.
    '''
    def setUp(self):
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg")

    def test_help_call_without_config(self):
        # simulate 'help' call
        sys.argv = ['opensubmit-exec', 'help']
        cmdline.console_script()

    def test_unlock_call_without_lock(self):
        # simulate 'unlock' call without existing lock
        sys.argv = ['opensubmit-exec', 'unlock']
        cmdline.console_script()

    def test_unlock_call_with_lock(self):
        # simulate 'unlock' call with existing lock
        sys.argv = ['opensubmit-exec', 'unlock']
        with locking.ScriptLock(self.config):
            cmdline.console_script()


class Library(SubmitStudentScenarioTestCase):
    '''
    Tests for the executor library functions used by the validator script.
    '''
    def setUp(self):
        settings.MAIN_URL = self.live_server_url
        super(Library, self).setUp()
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg",
            override_url=self.live_server_url)

    def _register_executor(self):
        server.send_hostinfo(self.config)
        return TestMachine.objects.order_by('-last_contact')[0]

    def _run_executor(self):
        return cmdline.download_and_run(self.config)

    def test_send_result(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)

        msg = uccrap + "Message from validation script."

        job.send_pass_result(info_student=msg)

        db_entries = SubmissionTestResult.objects.filter(
            kind=SubmissionTestResult.VALIDITY_TEST)

        self.assertEqual(db_entries[0].result, msg)

    def test_fetch_job_renaming(self):
        test_machine = self._register_executor()
        self.validated_assignment.test_machines.add(test_machine)
        with open(rootdir + "submfiles/validation/1000fff/helloworld.c", 'rb') as f:
            self.c.post('/assignments/%s/new/' % self.validated_assignment.pk, {
                'notes': 'This is a test submission.',
                'authors': str(self.user.pk),
                'attachment': f
            })
        job = server.fetch_job(self.config)
        self.assertIn('helloworld.c', os.listdir(job.working_dir))

    def test_grep(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)
        self.assertListEqual(job.grep("World"), ['helloworld.c'])
        self.assertListEqual(job.grep("foobar"), [])

    def test_ensure_files(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)
        self.assertTrue(job.ensure_files(['helloworld.c']))

    def test_job_attributes(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        self.assertNotEquals(None, job)
        self.assertEquals(job.timeout, self.validated_assignment.attachment_test_timeout)
        self.assertEquals(job.sub_id, str(sub.pk))
        self.assertEquals(job.file_id, str(sf.pk))
        self.assertEquals(job.submitter_name, sub.submitter.get_full_name())
        self.assertEquals(job.submitter_student_id, str(sub.submitter.profile.student_id))
        self.assertEquals(job.submitter_studyprogram, str(sub.submitter.profile.study_program))
        self.assertEquals(job.course, str(self.course))
        self.assertEquals(job.assignment, str(self.validated_assignment))

    def test_wrong_compile_call(self):
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        job = server.fetch_job(self.config)
        # Missing output
        self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, inputs=['input.c'])
        # Missing input
        self.assertRaises(exceptions.ValidatorBrokenException, job.run_compiler, output='output')
        # Wrong file, determined by compiler
        self.assertRaises(exceptions.WrongExitStatusException, job.run_compiler, inputs=['foo.c'], output='output')

    def test_cleanup(self):
        '''
        If configured, then the executor should remove all temporary
        files after finishing.
        '''
        for do_cleanup in [False, True]:
            sf = create_submission_file()
            sub = create_validatable_submission(
                self.user, self.validated_assignment, sf)
            test_machine = self._register_executor()
            sub.assignment.test_machines.add(test_machine)
            job = server.fetch_job(self.config)
            self.assertNotEquals(None, job)
            working_dir = job.working_dir
            job._config.set("Execution", "cleanup", str(do_cleanup))
            job._run_validate()
            if do_cleanup:
                with self.assertRaises(FileNotFoundError):
                    os.listdir(working_dir)
            else:
                self.assertNotEqual(os.listdir(working_dir), [])



class Validation(TestCase):
    '''
    Tests for the execution of validation scripts by the executor.
    '''

    def setUp(self):
        super(Validation, self).setUp()
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg")

    def _test_validation_case(self, directory):
        '''
        Each of the validator.py files uses the Python assert()
        statement to check by itself if the result is the expected
        one for its case.
        '''
        base_dir = os.path.dirname(__file__) + '/submfiles/validation/'
        cmdline.copy_and_run(self.config, base_dir + directory)

    def test_0100fff(self):
        self._test_validation_case('0100fff')

    def test_0100tff(self):
        self._test_validation_case('0100tff')

    def test_0100ttf(self):
        self._test_validation_case('0100ttf')

    def test_1000fff(self):
        self._test_validation_case('1000fff')

    def test_1000fft(self):
        self._test_validation_case('1000fft')

    def test_1000tff(self):
        self._test_validation_case('1000tff')

    def test_1000tft(self):
        self._test_validation_case('1000tft')

    def test_1000ttf(self):
        self._test_validation_case('1000ttf')

    def test_1000ttt(self):
        self._test_validation_case('1000ttt')

    def test_1010tff(self):
        self._test_validation_case('1010tff')

    def test_1010ttf(self):
        self._test_validation_case('1010ttf')

    def test_1100tff(self):
        self._test_validation_case('1100tff')

    def test_1100ttf(self):
        self._test_validation_case('1100ttf')

    def test_3000tff(self):
        self._test_validation_case('3000tff')

    def test_3000ttf(self):
        self._test_validation_case('3000ttf')

    def test_3010tff(self):
        self._test_validation_case('3010tff')

    def test_3010ttf(self):
        self._test_validation_case('3010ttf')

    def test_b000tff(self):
        self._test_validation_case('b000tff')

    def test_b010tff(self):
        self._test_validation_case('b010tff')

    def test_1000tfm(self):
        self._test_validation_case('1000tfm')

    def test_regressions(self):
        self._test_validation_case('regression_001')


class Communication(SubmitStudentScenarioTestCase):
    '''
    Tests for the communication of the executor with the OpenSubmit server.
    '''

    def setUp(self):
        settings.MAIN_URL = self.live_server_url
        super(Communication, self).setUp()
        self.config = config.read_config(
            os.path.dirname(__file__) + "/executor.cfg",
            override_url=self.live_server_url)

    def _register_executor(self):
        server.send_hostinfo(self.config)
        return TestMachine.objects.order_by('-last_contact')[0]

    def _run_executor(self):
        return cmdline.download_and_run(self.config)

    def _register_test_machine(self):
        '''
        Utility step for a common test case preparation:
        - Create validatable submission
        - Register a test machine for it
        '''
        sf = create_submission_file()
        sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        return sub

    def test_register_executor_explicit(self):
        machine_count = TestMachine.objects.all().count()
        assert(self._register_executor().pk)
        self.assertEqual(machine_count + 1, TestMachine.objects.all().count())

    @override_settings(JOB_EXECUTOR_SECRET='foo')
    def test_invalid_secret(self):
        self.assertNotEqual(True, self._run_executor())

    def test_everything_already_tested(self):
        create_validated_submission(self.user, self.validated_assignment)
        assert(self._register_executor().pk)
        self.assertEqual(False, self._run_executor())

    def test_parallel_executors_test(self):
        NUM_PARALLEL = 3
        self.validated_assignment.test_machines.add(self._register_executor())
        subs = []
        for i in range(1, NUM_PARALLEL + 1):
            stud = create_user(get_student_dict(i))
            self.course.participants.add(stud.profile)
            self.course.save()
            sf = create_submission_file()
            subs.append(create_validatable_submission(
                stud, self.validated_assignment, sf))

        # Span a number of threads, each triggering the executor
        # This only creates a real test case if executor serialization
        # is off (see tests/executor.cfg)
        return_codes = utils.run_parallel(len(subs), self._run_executor)
        self.assertEqual(
            len(list(filter((lambda x: x is True), return_codes))),
            len(subs))

        for sub in subs:
            results = SubmissionTestResult.objects.filter(
                kind=SubmissionTestResult.VALIDITY_TEST
            )
            self.assertEqual(NUM_PARALLEL, len(results))
            self.assertNotEqual(0, len(results[0].result))

    def test_too_long_validation(self):

This code seems to be duplicated in your project.

        from django.core import mail

        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, grading, "/submfiles/validation/d000fff/", "validator_run.py")
        assignment.attachment_test_timeout = 1
        assignment.save()
        sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        # Fire up the executor, should mark the submission as timed out
        self.assertEqual(True, self._run_executor())
        # Check if timeout marking took place
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
        text = sub.get_validation_result().result
        self.assertIn("took too long", text)
        # Check mail outbox for student information
        self.assertEqual(1, len(mail.outbox))
        for email in mail.outbox:
            self.assertIn("Validation failed", email.subject)
            self.assertIn("failed", email.body)
            self.assertIn("localhost", email.body)

    def test_broken_validator_feedback(self):

This code seems to be duplicated in your project.

        from django.core import mail

        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, grading, "/submfiles/validation/1000tfm/", "validator.zip")
        assignment.save()
        sf = create_submission_file("/submfiles/validation/1000tfm/packed.zip")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        # Fire up the executor
        self.assertEqual(False, self._run_executor())
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.TEST_VALIDITY_FAILED)
        text = sub.get_validation_result().result
        self.assertIn("Internal error", text)
        # Check mail outbox for student information
        self.assertEqual(1, len(mail.outbox))
        for email in mail.outbox:
            self.assertIn("Validation failed", email.subject)
            self.assertIn("failed", email.body)
            self.assertIn("localhost", email.body)

    def test_output_logging(self):

This code seems to be duplicated in your project.

        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, grading, "/submfiles/validation/1000fff/", "validator.py")
        assignment.save()
        sf = create_submission_file("/submfiles/validation/1000fff/helloworld.c")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)
        # Fire up the executor
        self.assertEqual(True, self._run_executor())
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.SUBMITTED_TESTED)
        text = sub.get_validation_result().result
        self.assertIn("quick brown fox", text)
        self.assertIn("provide your input", text)

    def test_too_long_full_test(self):

This code seems to be duplicated in your project.

        grading = create_pass_fail_grading()
        assignment = create_validated_assignment(
            self.course, 
            grading, 
            "/submfiles/validation/d000fff/",
            "validator_build.py",
            "validator_run.py")
        assignment.attachment_test_timeout = 1
        assignment.save()
        sf = create_submission_file("/submfiles/validation/d000fff/helloworld.c")
        sub = create_validatable_submission(
            self.user, assignment, sf)
        test_machine = self._register_executor()
        sub.assignment.test_machines.add(test_machine)

        # Fire up the executor for validation
        self.assertEqual(True, self._run_executor())
        # Fire up the executor for full test
        self.assertEqual(True, self._run_executor())
        # Check if timeout marking took place
        sub.refresh_from_db()
        self.assertEqual(sub.state, Submission.TEST_FULL_FAILED)
        assert("timeout" in sub.get_fulltest_result().result_tutor)
        # Failed full tests shall not be reported
        self.assertEqual(0, len(mail.outbox))

    def test_single_file_validator_test(self):
        sub = self._register_test_machine()

        self.assertEqual(True, self._run_executor())
        results = SubmissionTestResult.objects.filter(
            submission_file=sub.file_upload,
            kind=SubmissionTestResult.VALIDITY_TEST
        )
        self.assertEqual(1, len(results))

        self.assertEqual(True, self._run_executor())
        results = SubmissionTestResult.objects.filter(
            submission_file=sub.file_upload,
            kind=SubmissionTestResult.VALIDITY_TEST
        )
        self.assertEqual(1, len(results))
        self.assertNotEqual(0, len(results[0].result))
        # Graded assignment, so no mail at this stage
        self.assertEqual(0, len(mail.outbox))

    def test_ungraded_validation_email(self):

This code seems to be duplicated in your project.

        from django.core import mail

        sf = create_submission_file()
        assign = create_validated_assignment_with_file(self.course, None)
        sub = create_validatable_submission(self.user, assign, sf)
        test_machine = self._register_executor()
        assign.test_machines.add(test_machine)
        # has no grading scheme, so it is not graded
        assert(not sub.assignment.is_graded())

        # Fire up the executor for validation
        self.assertEqual(True, self._run_executor())

        # Fire up the executor for full test
        self.assertEqual(True, self._run_executor())

        # Check mail outbox for student information
        self.assertEqual(1, len(mail.outbox))
        for email in mail.outbox:
            self.assertIn("Validation successful", email.subject)
            self.assertIn("passed", email.body)
            self.assertIn("localhost", email.body)

    def test_full_test(self):
        sub = self._register_test_machine()
        # validation test
        self.assertEqual(True, self._run_executor())
        # full test
        self.assertEqual(True, self._run_executor())
        results = SubmissionTestResult.objects.filter(
            submission_file=sub.file_upload,
            kind=SubmissionTestResult.FULL_TEST
        )
        self.assertEqual(1, len(results))
        self.assertNotEqual(0, len(results[0].result_tutor))
        self.assertEqual(None, results[0].result)

    def test_inconsistent_state_email(self):
        '''
        Test operator email on inconsistent state.
        Since executor execution and submission sending is one step,
        we need to mock the incoming invalid executor request.
        '''
        sf = create_submission_file()
        self.sub = create_validatable_submission(
            self.user, self.validated_assignment, sf)
        test_machine = self._register_executor()
        self.sub.assignment.test_machines.add(test_machine)
        self.sub.state = Submission.TEST_FULL_PENDING
        self.sub.save()
        post_data = {'Secret': settings.JOB_EXECUTOR_SECRET,
                     'UUID': test_machine.host,
                     'Action': 'test_compile',
                     'SubmissionFileId': self.sub.file_upload.pk,
                     'PerfData': '',
                     'ErrorCode': 0,
                     'Message': 'In A Bottle'}
        self.c.post(reverse('jobs'), post_data)
        self.assertNotEqual(len(mail.outbox), 0)
        self.assertIn('Action reported by the executor: test_compile',
                      mail.outbox[0].message().as_string())

    def test_assignment_specific_test_machine(self):
        # Register two test machines T1 and T2
        real_machine = self._register_executor()
        fake_machine = TestMachine(host="127.0.0.2")
        fake_machine.save()
        # Assign each of them to a different assignment
        self.open_assignment.test_machines.add(real_machine)
        self.validated_assignment.test_machines.add(fake_machine)
        # Produce submission for the assignment linked to fake_machine
        sub1 = Submission(
            assignment=self.validated_assignment,
            submitter=self.user,
            state=Submission.TEST_VALIDITY_PENDING,
            file_upload=create_submission_file()
        )
        sub1.save()
        # Run real_machine executor, should not react on this submission
        old_sub1_state = sub1.state
        self.assertEqual(False, self._run_executor())
        # Make sure that submission object was not touched,
        # whatever the executor says
        sub1 = Submission.objects.get(pk=sub1.pk)
        self.assertEqual(old_sub1_state, sub1.state)