diff --git a/python3_contest_problem/__tester.py b/python3_contest_problem/__tester.py new file mode 100644 index 0000000..78a1dd2 --- /dev/null +++ b/python3_contest_problem/__tester.py @@ -0,0 +1,347 @@ +"""The generic (multi-language) main testing class that does all the work - trial compile, style checks, + run and grade. +""" +from __resulttable import ResultTable +import html +import os +import re +import __languagetask as languagetask +import base64 + + +# Values of QUESTION.precheck field +PRECHECK_DISABLED = 0 +PRECHECK_EMPTY = 1 +PRECHECK_EXAMPLES = 2 +PRECHECK_SELECTED = 3 +PRECHECK_ALL = 4 + +# Values of testtype +TYPE_NORMAL = 0 +TYPE_PRECHECKONLY = 1 +TYPE_BOTH = 2 + +# Global message for when a test-suite timeout occurs +TIMEOUT_MESSAGE = """A timeout occurred when running the whole test suite as a single program. +This is usually due to an endless loop in your code but can also arise if your code is very inefficient +and the accumulated time over all tests is excessive. Please ask a tutor or your lecturer if you need help +with making your program more efficient.""" + + +def get_jpeg_b64(filename): + """Return the contents of the given file (assumed to be jpeg) as a base64 + encoded string in utf-8. + """ + with open(filename, 'br') as fin: + contents = fin.read() + + return base64.b64encode(contents).decode('utf8') + + +class Tester: + def __init__(self, params, testcases): + """Initialise the instance, given the test of template and global parameters plus + all the testcases. Parameters required by this base class and all subclasses are: + 'STUDENT_ANSWER': code submitted by the student + 'SEPARATOR': the string to be used to separate tests in the output + 'ALL_OR_NOTHING: true if grading is all-or-nothing + 'stdinfromextra': true if the test-case 'extra' field is to be used for + standard input rather than the usual stdin field + 'runtestssingly': true to force a separate run for each test case + 'stdinfromextra': true if the extra field is used for standard input (legacy use only) + 'testisbash': true if tests are bash command line(s) rather than the default direct execution + of the compiled program. This can be used to supply command line arguments. + + """ + self.student_answer = self.clean(params['STUDENT_ANSWER']) + self.separator = params['SEPARATOR'] + self.all_or_nothing = params['ALL_OR_NOTHING'] + self.params = params + self.testcases = self.filter_tests(testcases) + self.result_table = ResultTable(params) + self.result_table.set_header(self.testcases) + + # It is assumed that in general subclasses will prefix student code by a prelude and + # postfix it by a postlude. + self.prelude = '' + self.prelude_length = 0 + self.postlude = '' + + self.task = None # SUBCLASS MUST DEFINE THIS + + def filter_tests(self, testcases): + """Return the relevant subset of the question's testcases. + This will be all testcases not marked precheck-only if it's not a precheck or all testcases if it is a + precheck and the question precheck is set to "All", or the appropriate subset in all other cases. + """ + if not self.params['IS_PRECHECK']: + return [test for test in testcases if test.testtype != TYPE_PRECHECKONLY] + elif self.params['QUESTION_PRECHECK'] == PRECHECK_ALL: + return testcases + elif self.params['QUESTION_PRECHECK'] == PRECHECK_EMPTY: + return [] + elif self.params['QUESTION_PRECHECK'] == PRECHECK_EXAMPLES: + return [test for test in testcases if test.useasexample] + elif self.params['QUESTION_PRECHECK'] == PRECHECK_SELECTED: + return [test for test in testcases if test.testtype in [TYPE_PRECHECKONLY, TYPE_BOTH]] + + def style_errors(self): + """Return a list of all the style errors. Implementation is language dependent. + Default is no style checking. + """ + return [] + + def single_program_build_possible(self): + """Return true if and only if the current configuration permits a single program to be + built and tried containing all tests. It should be true for write-a-program questions and + conditionally true for other types of questions that allow a "combinator" approach, + dependent on the presence of stdins in tests and other such conditions. + """ + raise NotImplementedError("Tester must have a single_program_build_possible method") + + def adjust_error_line_nums(self, error): + """Given a runtime error message, adjust it as may be required by the + language, e.g. adjusting line numbers + """ + raise NotImplementedError("Tester must have an adjust_error_line_nums method") + + def single_run_possible(self): + """Return true if a single program has been built and it is possible to use that in a single run + with all tests. + """ + return (self.task.executable_built + and not self.params['runtestssingly'] + and not self.result_table.has_stdins + and not self.params['testisbash']) + + def make_test_postlude(self, testcases): + """Return the postlude testing code containing all the testcode from + the given list of testcases (which may be the full set or a singleton list). + A separator must be printed between testcase outputs.""" + raise NotImplementedError("Tester must have a make_test_postlude method") + + def trial_compile(self): + """This function is the first check on the syntactic correctness of the submitted code. + It is called before any style checks are done. For compiled languages it should generally + call the standard language compiler on the student submitted code with any required prelude + added and, if possible, all tests included. CompileError should be raised if the compile fails, + which will abort all further testing. + If possible a complete ready-to-run executable should be built as well; if this succeeds, the + LanguageTasks 'executable_built' attribute should be set. This should be possible for write-a-program + questions or for write-a-function questions when there is no stdin data in any of the tests. + + Interpreted languages should perform what syntax checks are possible using the standard language tools. + If those checks succeeded, they should also attempt to construct a source program that incorporates all + the different tests (the old "combinator" approach) and ensure the task's 'executable_built' attribute + is True. + + The following implementation is sufficient for standard compiled languages like C, C++, Java. It + may need overriding for other languages. + """ + if self.single_program_build_possible(): + self.setup_for_test_runs(self.testcases) + make_executable = True + else: + self.postlude = '' + self.task.set_code(self.prelude + self.student_answer, self.prelude_length) + make_executable = False + + self.task.compile(make_executable) # Could raise CompileError + + def setup_for_test_runs(self, tests): + """Set the code and prelude length as appropriate for a run with all the given tests. May be called with + just a singleton list for tests if single_program_build_possible has returned false or if testing with + multiple tests has given exceptions. + This implementation may need to be overridden, e.g. if the student code should follow the test code, as + say in Matlab scripts. + """ + self.postlude = self.make_test_postlude(tests) + self.task.set_code(self.prelude + self.student_answer + self.postlude, self.prelude_length) + + def run_all_tests(self): + """Run all the tests, leaving self.ResultTable object containing all test results. + Can raise CompileError or RunError if things break. + If any runtime errors occur on the full test, drop back to running tests singly. + """ + done = False + if self.single_run_possible(): + # We have an executable ready to go, with no stdins or other show stoppers + output, error = self.task.run_code() + output = output.rstrip() + '\n' + error = error.strip() + '\n' + + # Generate a result table using all available test data. + results = output.split(self.separator + '\n') + errors = error.split(self.separator + '\n') + if len(results) == len(errors): + merged_results = [] + for result, error in zip(results, errors): + result = result.rstrip() + '\n' + if error.strip(): + adjusted_error = self.adjust_error_line_nums(error.rstrip()) + result += '\n*** RUN ERROR ***\n' + adjusted_error + merged_results.append(result) + + missed_tests = len(self.testcases) - len(merged_results) + + for test, output in zip(self.testcases, merged_results): + self.result_table.add_row(test, output) + + self.result_table.tests_missed(missed_tests) + if self.task.timed_out: + self.result_table.record_global_error(TIMEOUT_MESSAGE) + done = True + + if not done: + # Something broke. We will need to run each test case separately + self.task.executable_built = False + self.result_table.reset() + + if not done: + # If a single run isn't appropriate, do a separate run for each test case. + build_each_test = not self.task.executable_built + for i_test, test in enumerate(self.testcases): + if build_each_test: + self.setup_for_test_runs([test]) + self.task.compile(True) + standard_input = test.extra if self.params['stdinfromextra'] else test.stdin + if self.params['testisbash']: + output, error = self.task.run_code(standard_input, test.testcode) + else: + output, error = self.task.run_code(standard_input) + adjusted_error = self.adjust_error_line_nums(error.rstrip()) + self.result_table.add_row(test, output, adjusted_error) + if error and self.params['abortonerror']: + self.result_table.tests_missed(len(self.testcases) - i_test - 1) + break + + def compile_and_run(self): + """Phase one of the test operation: do a trial compile and then, if all is well and it's not a precheck, + continue on to run all tests. + Return a tuple mark, errors where mark is a fraction in 0 - 1 and errors is a list of all the errors. + self.test_results contains all the test details. + """ + mark = 0 + errors = [] + + # Do a trial compile, then a style check. If all is well, run the code + try: + self.trial_compile() + + if not self.params['nostylechecks']: + errors = self.style_errors() + if not errors: + if self.params['IS_PRECHECK'] and self.params['QUESTION_PRECHECK'] <= 1: + mark = 1 + else: + self.run_all_tests() + max_mark = sum(test.mark for test in self.testcases) + mark = self.result_table.get_mark() / max_mark # Fractional mark 0 - 1 + except languagetask.CompileError as err: + adjusted_error = self.adjust_error_line_nums(str(err).rstrip()) + errors.append("COMPILE ERROR\n" + adjusted_error) + except languagetask.RunError as err: + adjusted_error = self.adjust_error_line_nums(str(err).rstrip()) + errors.append('RUN ERROR\n' + adjusted_error) + return mark, errors + + def prerun_hook(self): + """A hook for subclasses to do initial setup or code hacks etc + Returns a list of errors, to which other errors are appended + """ + return [] + + def get_all_images_html(self): + r"""Search the current directory for images named _image.*(Expected|Got)(\d+).png. + For each such file construct an html img element with the data encoded + in a dataurl. + If we're running the sample answer, always return [] - images will be + picked up when we run the actual answer. + Returns a list of tuples (img_elements, column_name, row_number) where + column_name is either 'Expected' or 'Got', defining in which result table + column the image belongs and row number is the row (0-origin, excluding + the header row). + """ + images = [] + if self.params.get('running_sample_answer', False): + return [] + if self.params['imagewidth'] is not None: + width_spec = " width={}".format(self.params['imagewidth']) + else: + width_spec = "" + files = sorted(os.listdir('.')) + for filename in files: + match = re.match(r'_image[^.]*\.(Expected|Got)\.(\d+).png', filename) + if match: + image_data = get_jpeg_b64(filename) + img_template = '' + img_html = img_template.format(width_spec, image_data) + column = match.group(1) # Name of column + row = int(match.group(2)) # 0-origin row number + images.append((img_html, column, row)) + return images + + def test_code(self): + """The "main program" for testing. Returns the test outcome, ready to be printed by json.dumps""" + errors = self.prerun_hook() + if errors: + mark = 0 + else: + mark, errors = self.compile_and_run() + + outcome = {"fraction": mark} + + error_text = '\n'.join(errors) + # TODO - check if error line numbers are still being corrected in C and matlab + if self.params['IS_PRECHECK']: + if mark == 1: + prologue = "

Passed 🙂

" + else: + prologue = "

Failed, as follows.

" + elif errors: + prologue = "

Pre-run checks failed

\n" + else: + prologue = "" + + if prologue: + outcome['prologuehtml'] = prologue + self.htmlize(error_text) + + epilogue = '' + images = self.get_all_images_html() + if images: + for (image, column, row) in images: + self.result_table.add_image(image, column, row) + outcome['columnformats'] = self.result_table.get_column_formats() + + if len(self.result_table.table) > 1: + outcome['testresults'] = self.result_table.get_table() + outcome['showdifferences'] = True + + if self.result_table.global_error: + epilogue += "

Run Error

{}
".format( + self.htmlize(self.result_table.global_error)) + + if self.result_table.aborted: + epilogue = outcome.get('epiloguehtml', '') + ( + "
Testing was aborted due to runtime errors.
") + + if self.result_table.missing_tests != 0: + template = "
{} tests not run due to previous errors.
" + epilogue += template.format(self.result_table.missing_tests) + + if self.result_table.failed_hidden: + epilogue += "
One or more hidden tests failed.
" + + if epilogue: + outcome['epiloguehtml'] = epilogue + return outcome + + @staticmethod + def clean(code): + """Return the given code with trailing white space stripped from each line""" + return '\n'.join([line.rstrip() for line in code.split('\n')]) + '\n' + + @staticmethod + def htmlize(message): + """An html version of the given error message""" + return '
' + html.escape(message) + '
' if message else '' diff --git a/python3_contest_problem/pytester.py b/python3_contest_problem/pytester.py new file mode 100644 index 0000000..ff3fe65 --- /dev/null +++ b/python3_contest_problem/pytester.py @@ -0,0 +1,279 @@ +"""The main python-program testing class that does all the work - style checks, + run and grade. A subclass of the generic tester. + Since each test can by run within the current instance of Python using + an exec, we avoid the usual complication of combinators by running + each test separately regardless of presence of stdin, testcode, etc. +""" +import __pytask as pytask +import re +from __tester import Tester +from __pystylechecker import StyleChecker +from random import randint + + +class PyTester(Tester): + def __init__(self, params, testcases): + """Initialise the instance, given the test of template and global parameters plus + all the testcases. Parameters relevant to this class are all those listed for the Tester class plus + 'extra' which takes the values 'pretest' or 'posttest' (the other possible value, 'stdin', has been + handled by the main template). + Additionally the support classes like stylechecker and pyparser need their + own params - q.v. + """ + super().__init__(params, testcases) # Most of the task is handed by the generic tester + + # Py-dependent attributes + self.task = pytask.PyTask(params) + self.prelude = '' + + if params['isfunction']: + if not self.has_docstring(): + self.prelude = '"""Dummy docstring for a function"""\n' + + if params['usesmatplotlib']: + self.prelude += '\n'.join([ + 'import os', + 'import matplotlib as _mpl', + '_mpl.use("Agg")', + 'from __plottools import print_plot_info', + ]) + '\n' + self.params['pylintoptions'].append("--disable=ungrouped-imports") + + if params['usesnumpy']: + self.prelude += 'import numpy as np\n' + self.params['pylintoptions'].append("--disable=unused-import,ungrouped-imports") + self.params['pylintoptions'].append("--extension-pkg-whitelist=numpy") + + for import_string in params['imports']: + if ' ' not in import_string: + self.prelude += 'import ' + import_string + '\n' + else: + self.prelude += import_string + '\n' + + if params['prelude'] != '': + self.prelude += '\n' + params['prelude'].rstrip() + '\n' + + try: + with open('_prefix.py') as prefix: + prefix_code = prefix.read() + self.prelude += prefix_code.rstrip() + '\n' + + except FileNotFoundError: + pass + + self.prelude_length = len(self.prelude.splitlines()) + if self.has_docstring() and self.prelude_length > 0: + # If we insert prelude in front of the docstring, pylint will + # give a missing docstring error. Our horrible hack solution is + # to insert an extra docstring at the start and turn off the + # resulting 'string statement has no effect' error. + self.prelude = '"""Dummy docstring for a function"""\n' + self.prelude + self.prelude_length += 1 + self.params['pylintoptions'].append("--disable=W0105") + self.style_checker = StyleChecker(self.prelude, self.params['STUDENT_ANSWER'], self.params) + + def has_docstring(self): + """True if the student answer has a docstring, which means that, + when stripped, it starts with a string literal. + """ + prog = self.params['STUDENT_ANSWER'].lstrip() + return prog.startswith('"') or prog.startswith("'") + + + def tweaked_warning(self, message): + """Improve the warning message by updating line numbers and replacing : with Line + """ + return self.adjust_error_line_nums(message).replace(':', 'Line ') + + + def style_errors(self): + """Return a list of all the style errors. Start with local tests and continue with pylint + only if there are no local errors. + """ + errors = [] + if self.params.get('localprechecks', True): + try: + errors += self.style_checker.local_errors() # Note: prelude not included so don't adjust line nums + except Exception as e: + errors += [str(e)] + else: + check_for_passive = (self.params['warnifpassiveoutput'] and self.params['isfunction']) + if check_for_passive: + passive = self.passive_output() + warning_messages = [line for line in passive.splitlines() if 'Warning:' in line] + if warning_messages: + errors += [self.tweaked_warning(message) for message in warning_messages] + elif passive: + errors.append("Your code was not expected to generate any output " + + "when executed stand-alone.\nDid you accidentally include " + + "your test code?\nOr you might have a wrong import statement - have you tested in Wing?") + errors.append(passive) + + if len(errors) == 0 or self.params.get('forcepylint', False): + # Run precheckers (pylint, mypy) + try: + # Style-check the program without any test cases or other postlude added + errors += self.style_checker.style_errors() + except Exception as e: + error_text = '*** Unexpected error while running precheckers. Please report ***\n' + str(e) + errors += [error_text] + errors = [self.simplify_error(self.adjust_error_line_nums(error)) for error in errors] + errors = [error for error in errors if not error.startswith('************* Module')] + + errors = [error.replace(', ', '') for error in errors] # Another error tidying operation + if errors: + errors.append("\nSorry, but your code doesn't pass the style checks.") + return errors + + def prerun_hook(self): + """A hook for subclasses to do initial setup or code hacks etc + Returns a list of errors, to which other errors are appended. + In this class we use it firstly to check that the number of Prechecks + allowed has not been exceeded and then, of not, to perform + required hacks to disable calls to main. If the call to main_hacks + fails, assume the code is bad and will get flagged by pylint in due course. + """ + step_info = self.params['STEP_INFO'] + max_prechecks = self.params.get('maxprechecks', None) + if max_prechecks and step_info['numprechecks'] >= max_prechecks: + return [f"Sorry, you have reached the limit on allowed prechecks ({max_prechecks}) for this question."] + + try: + return self.main_hacks() + except: + return [] + + def passive_output(self): + """ Return the passive output from the student answer code + This is essentially a "dry run" of the code. + """ + code = self.prelude + self.params['STUDENT_ANSWER'] + if self.params['usesmatplotlib']: + code += '\n'.join([ + 'figs = _mpl.pyplot.get_fignums()', + 'if figs:', + ' print(f"{len(figs)} figures found")', + ' print(f"{_mpl.pyplot.get_figlabels()}")' + ]) + '\n' + task = pytask.PyTask(self.params, code) + with open(f"WTF{randint(0,100)}.py", 'w') as outfile: + outfile.write(code) + task.compile() + captured_output, captured_error = task.run_code() + return (captured_output + '\n' + captured_error).strip() + + def make_test_postlude(self, testcases): + """Return the code that follows the student answer containing all the testcode + from the given list of testcases, which should always be of length 1 + (because we don't bother trying to combine all the tests into a + single run in Python) + """ + assert len(testcases) == 1 + if self.params['notest']: + return '' + test = testcases[0] + tester = '' + if self.params['globalextra'] and self.params['globalextra'] == 'pretest': + tester += self.params['GLOBAL_EXTRA'] + '\n' + if test.extra and self.params['extra'] == 'pretest': + tester += test.extra + '\n' + if test.testcode: + tester += test.testcode.rstrip() + '\n' + if self.params['globalextra'] and self.params['globalextra'] == 'posttest': + tester += self.params['GLOBAL_EXTRA'] + '\n' + if test.extra and self.params['extra'] == 'posttest': + tester += test.extra + '\n' + + if self.params['usesmatplotlib']: + if 'dpi' in self.params and self.params['dpi']: + extra = f", dpi={self.params['dpi']}" + else: + extra = '' + if self.params.get('running_sample_answer', False): + column = 'Expected' + else: + column = 'Got' + test_num = len(self.result_table.table) - 1 # 0-origin test number from result table + tester += '\n'.join([ + 'figs = _mpl.pyplot.get_fignums()', + 'for fig in figs:', + ' _mpl.pyplot.figure(fig)', + ' row = {}'.format(test_num), + ' column = "{}"'.format(column), + ' _mpl.pyplot.savefig("_image{}.{}.{}.png".format(fig, column, row), bbox_inches="tight"' + '{})'.format(extra), + ' _mpl.pyplot.close(fig)' + ]) + '\n' + return tester + + def single_program_build_possible(self): + """We avoid all the complication of trying to run all tests in + a single subprocess run by using exec to run each test singly. + """ + return False + + def adjust_error_line_nums(self, error): + """Subtract the prelude length of all line numbers in the given error message + """ + error_patterns = [ + (r'(.*.* \(syntax-error\).*)', []), + (r'(.*File ".*", line +)(\d+)(, in .*)', [2]), + (r'(.*: *)(\d+)(, *\d+:.*\(.*line +)(\d+)(\).*)', [2, 4]), + (r'(.*: *)(\d+)(, *\d+:.*\(.*\).*)', [2]), + (r'(.*:)(\d+)(:\d+: [A-Z]\d+: .*line )(\d+)(.*)', [2, 4]), + (r'(.*:)(\d+)(:\d+: [A-Z]\d+: .*)', [2]), + (r'(.*:)(\d+)(: [a-zA-Z]*Warning.*)', [2]), + ] + output_lines = [] + for line in error.splitlines(): + for pattern, line_group_nums in error_patterns: + match = re.match(pattern, line) + if match: + line = '' + for i, group in enumerate(match.groups(), 1): + if i in line_group_nums: + linenum = int(match.group(i)) + adjusted = linenum - self.prelude_length + line += str(adjusted) + else: + line += group + break + + output_lines.append(line) + return '\n'.join(output_lines) + + def simplify_error(self, error): + """Return a simplified version of a pylint error with Line inserted in + lieu of __source.py:

: Xnnnn + """ + pattern = r'__source.py:(\d+): *\d+: *[A-Z]\d+: (.*)' + match = re.match(pattern, error) + if match: + return f"Line {match.group(1)}: {match.group(2)}" + else: + return error + + def main_hacks(self): + """Modify the code to be tested if params stripmain or stripmainifpresent' + are specified. Returns a list of errors encountered while so doing. + """ + errors = [] + if self.params['stripmain'] or self.params['stripmainifpresent']: + main_calls = self.style_checker.find_function_calls('main') + if self.params['stripmain'] and main_calls == []: + errors.append("No call to main() found") + else: + student_lines = self.student_answer.split('\n') + for (line, depth) in main_calls: + if depth == 0: + main_call = student_lines[line] + if not re.match(r' *main\(\)', main_call): + errors.append(f"Illegal call to main().\n" + + "main should not take any parameters and should not return anything.") + else: + student_lines[line] = main_call.replace( + "main", "pass # Disabled call to main") + else: + student_lines[line] += " # We've let you call main here." + self.params['STUDENT_ANSWER'] = self.student_answer = '\n'.join(student_lines) + '\n' + + return errors diff --git a/python3_deskcheck/junk.py b/python3_deskcheck/junk.py new file mode 100644 index 0000000..1a9249d --- /dev/null +++ b/python3_deskcheck/junk.py @@ -0,0 +1,2 @@ +for i in [1, 2, 3] + print(i) \ No newline at end of file diff --git a/python3_deskcheck/prototypeextra.html b/python3_deskcheck/prototypeextra.html index 411e05f..ee32d7e 100644 --- a/python3_deskcheck/prototypeextra.html +++ b/python3_deskcheck/prototypeextra.html @@ -10,6 +10,7 @@ + + + + + + Simple js-parsons example + + +

+

Answer:

+

+ The answer contains line(s) not available in the preload! +

+ +
+ +
+

Drag-able code lines:

+ + +
+ + +
+ +
+ +
+
+
+
+
+
+
+
+ + \ No newline at end of file diff --git a/python3_scratchpad/__plottools.py b/python3_scratchpad/__plottools.py index 29ecf05..848c7c4 100644 --- a/python3_scratchpad/__plottools.py +++ b/python3_scratchpad/__plottools.py @@ -102,14 +102,11 @@ def print_line(self, line, xsamples): """Print the info for the given line""" if self.params['show_colour']: print("Color:", self.normalise_colour(line.get_color())) - marker = str(line.get_marker()) - if marker == 'None': - marker = '' # Trying to achieve some form of consistency with linestyle here! + marker = line.get_marker() + if marker == '': + marker = None print("Marker:", marker) - linestyle = str(line.get_linestyle()) - if linestyle == 'None': - linestyle = '' # Hack to counter the Matplotlib hack! - print("Line style:", linestyle) + print("Line style:", line.get_linestyle()) label = line.get_label() if label and self.params['show_linelabels']: print("Label:", label) diff --git a/python3_scratchpad/__pytask.py b/python3_scratchpad/__pytask.py index 3cb56bf..bbb0dd0 100644 --- a/python3_scratchpad/__pytask.py +++ b/python3_scratchpad/__pytask.py @@ -103,14 +103,15 @@ def new_import(name, *args, **kwargs): # (mct63) Insure print always prints to the redirected stdout and not actual stdout. # (rjl83) Also keep track of print quantity and raise ExcessiveOutput if too much is generated. def new_print(*values, sep=' ', end='\n', file=None, flush=False): + string_values = [] for value in values: - try: - CodeTrap.output_chars += len(str(value)) - except: - pass + strvalue = str(value) + string_values.append(strvalue) + CodeTrap.output_chars += len(strvalue) + if CodeTrap.output_chars > self.params['maxoutputbytes']: raise ExcessiveOutput() - return print(*values, sep=sep, end=end, file=sys.stdout) + return print(*string_values, sep=sep, end=end, file=sys.stdout) # force 'input' to echo to stdin to stdout if self.params['echostandardinput']: diff --git a/python3_scratchpad/__resulttable.py b/python3_scratchpad/__resulttable.py index 9d4b168..2b6b2cb 100644 --- a/python3_scratchpad/__resulttable.py +++ b/python3_scratchpad/__resulttable.py @@ -14,13 +14,17 @@ class ResultTable: def __init__(self, params): self.params = params self.mark = 0 + self.max_mark = 0 self.table = None self.failed_hidden = False self.aborted = False self.has_stdins = False self.has_tests = False + self.has_extra = False + self.has_expected = False self.hiding = False self.num_failed_tests = 0 + self.num_failed_hidden_tests = 0 self.missing_tests = 0 self.global_error = '' self.column_formats = None @@ -42,9 +46,10 @@ def set_header(self, testcases): of various table columns. """ header = ['iscorrect'] + required_columns = {field: hdr for hdr, field in self.params['resultcolumns']} self.column_formats = ['%s'] - if any(test.testcode.strip() != '' for test in testcases): - header.append("Test") + if 'testcode' in required_columns and any(test.testcode.strip() != '' for test in testcases): + header.append(required_columns['testcode']) self.has_tests = True # If the test code should be rendered in html then set that as column format. if any(getattr(test, 'test_code_html', None) for test in testcases): @@ -53,12 +58,26 @@ def set_header(self, testcases): self.column_formats.append('%s') stdins = [test.extra if self.params['stdinfromextra'] else test.stdin for test in testcases] - if any(stdin.rstrip() != '' for stdin in stdins): - header.append('Input') + if 'stdin' in required_columns and any(stdin.rstrip() != '' for stdin in stdins): + header.append(required_columns['stdin']) self.column_formats.append('%s') self.has_stdins = True - header += ['Expected', 'Got', 'iscorrect', 'ishidden'] - self.column_formats += ['%s', '%s', '%s', '%s'] + + if 'extra' in required_columns: + header.append(required_columns['extra']) + self.column_formats.append('%s') + self.has_extra = True + + if 'expected' in required_columns: + header.append(required_columns['expected']) + self.column_formats.append('%s') + self.has_expected = True + + # Always include the got column, regardless of required_columns, + # as it can contain error messages, + + header += ['Got', 'iscorrect', 'ishidden'] + self.column_formats += ['%s', '%s', '%s'] self.table = [header] def image_column_nums(self): @@ -119,9 +138,17 @@ def add_row(self, testcase, result, error=''): row.append(testcase.test_code_html) else: row.append(testcase.testcode) + if self.has_stdins: row.append(testcase.extra if self.params['stdinfromextra'] else testcase.stdin) - row.append(testcase.expected.rstrip()) + + if self.has_extra: + row.append(testcase.extra.rstrip()) + + if self.has_expected: + row.append(testcase.expected.rstrip()) + + # Always include the result column as it may contain error messages. max_len = self.params.get('maxstringlength', MAX_STRING_LENGTH) result = sanitise(result.rstrip('\n'), max_len) @@ -133,12 +160,16 @@ def add_row(self, testcase, result, error=''): result = error_message row.append(result) + display = testcase.display.upper() + self.max_mark += testcase.mark if is_correct: self.mark += testcase.mark else: self.num_failed_tests += 1 + if display == 'HIDE': + self.num_failed_hidden_tests += 1 row.append(is_correct) - display = testcase.display.upper() + is_hidden = ( self.hiding or display == 'HIDE' or @@ -155,7 +186,16 @@ def add_row(self, testcase, result, error=''): self.aborted = True def get_mark(self): - return self.mark if self.num_failed_tests == 0 or not self.params['ALL_OR_NOTHING'] else 0 + if self.num_failed_tests == 0: + return self.mark + # Failed one or more tests + elif (self.num_failed_tests == self.num_failed_hidden_tests) and self.params['failhiddenonlyfract'] > 0: + return self.max_mark * self.params['failhiddenonlyfract'] + elif not self.params['ALL_OR_NOTHING']: + return self.mark + else: + return 0 + @staticmethod def htmlise(s): diff --git a/python3_scratchpad/__tester.py b/python3_scratchpad/__tester.py index 82e98f1..78a1dd2 100644 --- a/python3_scratchpad/__tester.py +++ b/python3_scratchpad/__tester.py @@ -252,7 +252,7 @@ def prerun_hook(self): return [] def get_all_images_html(self): - """Search the current directory for images named _image.*(Expected|Got)(\d+).png. + r"""Search the current directory for images named _image.*(Expected|Got)(\d+).png. For each such file construct an html img element with the data encoded in a dataurl. If we're running the sample answer, always return [] - images will be diff --git a/python3_scratchpad/pytester.py b/python3_scratchpad/pytester.py index c0876f5..fa7c98e 100644 --- a/python3_scratchpad/pytester.py +++ b/python3_scratchpad/pytester.py @@ -128,10 +128,16 @@ def style_errors(self): def prerun_hook(self): """A hook for subclasses to do initial setup or code hacks etc Returns a list of errors, to which other errors are appended. - In this class we use it to perform required hacks to disable - calls to main. If the call to main_hacks fails, assume the code - is bad and will get flagged by pylint in due course. + In this class we use it firstly to check that the number of Prechecks + allowed has not been exceeded and then, of not, to perform + required hacks to disable calls to main. If the call to main_hacks + fails, assume the code is bad and will get flagged by pylint in due course. """ + step_info = self.params['STEP_INFO'] + max_prechecks = self.params.get('maxprechecks', None) + if max_prechecks and step_info['numprechecks'] >= max_prechecks: + return [f"Sorry, you have reached the limit on allowed prechecks ({max_prechecks}) for this question."] + try: return self.main_hacks() except: @@ -239,7 +245,7 @@ def simplify_error(self, error): """Return a simplified version of a pylint error with Line inserted in lieu of __source.py:

: Xnnnn """ - pattern = f'__source.py:(\d+): *\d+: *[A-Z]\d+: (.*)' + pattern = r'__source.py:(\d+): *\d+: *[A-Z]\d+: (.*)' match = re.match(pattern, error) if match: return f"Line {match.group(1)}: {match.group(2)}" @@ -260,8 +266,8 @@ def main_hacks(self): for (line, depth) in main_calls: if depth == 0: main_call = student_lines[line] - if not re.match(' *main\(\)', main_call): - errors.append(f"Illegal call to main().\n" + + if not re.match(r' *main\(\)', main_call): + errors.append("Illegal call to main().\n" + "main should not take any parameters and should not return anything.") else: student_lines[line] = main_call.replace( diff --git a/python3_scratchpad/template.py b/python3_scratchpad/template.py index b79d70e..9fa6653 100644 --- a/python3_scratchpad/template.py +++ b/python3_scratchpad/template.py @@ -7,25 +7,23 @@ from pytester import PyTester -STANDARD_PYLINT_OPTIONS = ['--disable=trailing-whitespace,superfluous-parens,' + - 'bad-continuation,min-public-methods,too-few-public-methods,star-args,' + - 'unbalanced-tuple-unpacking,too-many-statements,' + - 'consider-using-enumerate,simplifiable-if-statement,' + - 'consider-iterating-dictionary,trailing-newlines,no-else-return,' + - 'consider-using-dict-comprehension,' + - 'len-as-condition,inconsistent-return-statements,consider-using-join,' + - 'singleton-comparison,unused-variable,chained-comparison,no-else-break,' + - 'consider-using-in,useless-object-inheritance,unnecessary-pass,' + - 'reimported,wrong-import-order,wrong-import-position,ungrouped-imports,' + - 'consider-using-set-comprehension,no-else-raise,' + - 'unspecified-encoding,use-dict-literal,consider-using-with,' + - 'duplicate-string-formatting-argument,consider-using-dict-items,' + - 'consider-using-max-builtin,unnecessary-lambda,consider-using-with,' + - 'consider-using-f-string,unspecified-encoding,use-dict-literal,' + - 'use-a-generator', - '--enable=C0326', - '--good-names=i,j,k,n,s,c,_' - ] +STANDARD_PYLINT_OPTIONS = ['--disable=trailing-whitespace,superfluous-parens,' + + 'too-few-public-methods,consider-using-f-string,' + + 'unbalanced-tuple-unpacking,too-many-statements,' + + 'consider-using-enumerate,simplifiable-if-statement,' + + 'consider-iterating-dictionary,trailing-newlines,no-else-return,' + + 'consider-using-dict-comprehension,consider-using-generator,' + + 'len-as-condition,inconsistent-return-statements,consider-using-join,' + + 'singleton-comparison,unused-variable,chained-comparison,no-else-break,' + + 'consider-using-in,useless-object-inheritance,unnecessary-pass,' + + 'reimported,wrong-import-order,wrong-import-position,ungrouped-imports,' + + 'consider-using-set-comprehension,no-else-raise,unnecessary-lambda-assignment,' + + 'unspecified-encoding,use-dict-literal,,consider-using-with,' + + 'duplicate-string-formatting-argument,consider-using-dict-items,' + + 'consider-using-max-builtin,use-a-generator,unidiomatic-typecheck', + '--good-names=i,j,k,n,s,c,_' + ] + locale.setlocale(locale.LC_ALL, 'C.UTF-8') @@ -48,6 +46,7 @@ 'maxfunctionlength': 30, 'maxnumconstants': 4, 'maxoutputbytes': 10000, + 'maxprechecks': None, 'maxstringlength': 2000, 'norun': False, 'nostylechecks': False, @@ -57,9 +56,10 @@ 'prelude': '', 'proscribedbuiltins': ['exec', 'eval'], 'proscribedfunctions': [], - 'proscribedconstructs': ["goto", "while_with_else"], + 'proscribedconstructs': ["goto"], 'proscribedsubstrings': [], 'pylintoptions': [], + 'pylintmatplotlib': False, 'requiredconstructs': [], 'requiredfunctiondefinitions': [], 'requiredfunctioncalls': [], @@ -78,8 +78,11 @@ 'importlib': { 'onlyallow': [] }, + 'fileinput': { + 'onlyallow': [] + }, 'os': { - 'disallow': ['system', '_exit', '_.*'] + 'disallow': ['system', '_exit', '_.*', 'open', 'fdopen', 'listdir'] }, 'subprocess': { 'onlyallow': [] @@ -143,6 +146,12 @@ def process_template_params(): else: PARAMS[param_name] = default; + result_columns = """{{QUESTION.resultcolumns}}""".strip(); + if result_columns == '': + PARAMS['resultcolumns'] = [['Test', 'testcode'], ['Input', 'stdin'], ['Expected', 'expected'], ['Got', 'got']] + else: + PARAMS['resultcolumns'] = json.loads(result_columns); + if PARAMS['extra'] == 'stdin': PARAMS['stdinfromextra'] = True if PARAMS['runextra']: @@ -153,7 +162,10 @@ def process_template_params(): if PARAMS['allowglobals']: PARAMS['pylintoptions'].append("--const-rgx='[a-zA-Z_][a-zA-Z0-9_]{2,30}$'") if PARAMS['usesmatplotlib']: - PARAMS['pylintoptions'].append("--disable=reimported,wrong-import-position,wrong-import-order,unused-import") + if PARAMS['pylintmatplotlib']: + PARAMS['pylintoptions'].append("--disable=reimported,wrong-import-position,wrong-import-order,unused-import") + else: + PARAMS['precheckers'] = [] if PARAMS['testisbash']: print("testisbash is not implemented for Python") @@ -190,8 +202,7 @@ def process_global_params(): """Plug into the PARAMS variable all the "global" parameters from the question and its answer (as distinct from the template parameters). """ - response = json.loads("""{{ STUDENT_ANSWER | e('py') }}""") - PARAMS['STUDENT_ANSWER'] = response['answer_code'][0].rstrip() + '\n' + PARAMS['STUDENT_ANSWER'] = """{{ STUDENT_ANSWER | e('py') }}""".rstrip() + '\n' PARAMS['SEPARATOR'] = "##" PARAMS['IS_PRECHECK'] = "{{ IS_PRECHECK }}" == "1" PARAMS['QUESTION_PRECHECK'] = {{ QUESTION.precheck }} # Type of precheck: 0 = None, 1 = Empty etc diff --git a/python3_scratchpad_plus/PROTOTYPE_python3_scratchpad_plus.hacking.html b/python3_scratchpad_plus/PROTOTYPE_python3_scratchpad_plus.hacking.html new file mode 100644 index 0000000..e963b95 --- /dev/null +++ b/python3_scratchpad_plus/PROTOTYPE_python3_scratchpad_plus.hacking.html @@ -0,0 +1,317 @@ + + + + +

+ + + +
Test table
+
+
+
+ + + + + + \ No newline at end of file diff --git a/python3_scratchpad_plus/PROTOTYPE_python3_scratchpad_plus.html b/python3_scratchpad_plus/PROTOTYPE_python3_scratchpad_plus.html new file mode 100644 index 0000000..998da85 --- /dev/null +++ b/python3_scratchpad_plus/PROTOTYPE_python3_scratchpad_plus.html @@ -0,0 +1,700 @@ + + + + +
+ + + +
Examples
+
+
+
+ + +
+
Answer
+ +
+ + + +â–¶ Scratchpad + + + + + + + + + + + + + + + + diff --git a/python3_scratchpad_plus/PROTOTYPE_python3_state_sequence_table_tester.xml b/python3_scratchpad_plus/PROTOTYPE_python3_state_sequence_table_tester.xml new file mode 100644 index 0000000..6e3d406 --- /dev/null +++ b/python3_scratchpad_plus/PROTOTYPE_python3_state_sequence_table_tester.xml @@ -0,0 +1,631 @@ + + + + + + PROTOTYPE_python3_state_sequence_table_tester + + + The prototype for a question type that asks a student to step through the execution of a given bit of code line-by-line, updating the program state at each step in a given table.

+

This is a skeleton rewrite of the SST question type in which the SST parameters that were previously set by template parameters are now set by a control panel within the question. Only the num_vars and num_rows parameters are implemented so far. All the others (input_width, output_width, return_width) are still to be done. And the control panel should be hidden from the student, visible only to the teacher.

+

To create a new question of this type:

+
    +
  1. Create a new coderunner question of type python3_state_sequence_table_tester.
    2. +
  2. Fill in the question name and question text and set the output for the first test case to OK.
    3. +
  3. Save.
    4. +
  4. Open the Answer Preload section.
    5. +
  5. Set the parameters in the answer to whatever you want.
    6. +
  6. Fill in the answer SST with the required values.
    7. +
  7. Save
    8. +
+

]]> + + + + + 1 + 0 + 0 + + python3_state_sequence_table_tester + 2 + 1 + 10, 20, ... + 0 + 0 + 0 + 10 + + + + 1 + + + 0 + 0 + + 0 + #\n|ms]]> + python3 + + + EqualityGrader + + + + + 1 + 1 + None + 0 + {} + 1 + html + + 0 + 0 + 10240 + + + 1 + 0 + +

SST parameters

+ num_vars: +
+ num_rows: +
+ input_width: +
+ output_width: +
+ return_width: +
+

+
+
+ + ]]> + + + + + + + + + + OK + + + + + + SHOW + + + + + + \ No newline at end of file diff --git a/python3_scratchpad_plus/__tester.py b/python3_scratchpad_plus/__tester.py new file mode 100644 index 0000000..78a1dd2 --- /dev/null +++ b/python3_scratchpad_plus/__tester.py @@ -0,0 +1,347 @@ +"""The generic (multi-language) main testing class that does all the work - trial compile, style checks, + run and grade. +""" +from __resulttable import ResultTable +import html +import os +import re +import __languagetask as languagetask +import base64 + + +# Values of QUESTION.precheck field +PRECHECK_DISABLED = 0 +PRECHECK_EMPTY = 1 +PRECHECK_EXAMPLES = 2 +PRECHECK_SELECTED = 3 +PRECHECK_ALL = 4 + +# Values of testtype +TYPE_NORMAL = 0 +TYPE_PRECHECKONLY = 1 +TYPE_BOTH = 2 + +# Global message for when a test-suite timeout occurs +TIMEOUT_MESSAGE = """A timeout occurred when running the whole test suite as a single program. +This is usually due to an endless loop in your code but can also arise if your code is very inefficient +and the accumulated time over all tests is excessive. Please ask a tutor or your lecturer if you need help +with making your program more efficient.""" + + +def get_jpeg_b64(filename): + """Return the contents of the given file (assumed to be jpeg) as a base64 + encoded string in utf-8. + """ + with open(filename, 'br') as fin: + contents = fin.read() + + return base64.b64encode(contents).decode('utf8') + + +class Tester: + def __init__(self, params, testcases): + """Initialise the instance, given the test of template and global parameters plus + all the testcases. Parameters required by this base class and all subclasses are: + 'STUDENT_ANSWER': code submitted by the student + 'SEPARATOR': the string to be used to separate tests in the output + 'ALL_OR_NOTHING: true if grading is all-or-nothing + 'stdinfromextra': true if the test-case 'extra' field is to be used for + standard input rather than the usual stdin field + 'runtestssingly': true to force a separate run for each test case + 'stdinfromextra': true if the extra field is used for standard input (legacy use only) + 'testisbash': true if tests are bash command line(s) rather than the default direct execution + of the compiled program. This can be used to supply command line arguments. + + """ + self.student_answer = self.clean(params['STUDENT_ANSWER']) + self.separator = params['SEPARATOR'] + self.all_or_nothing = params['ALL_OR_NOTHING'] + self.params = params + self.testcases = self.filter_tests(testcases) + self.result_table = ResultTable(params) + self.result_table.set_header(self.testcases) + + # It is assumed that in general subclasses will prefix student code by a prelude and + # postfix it by a postlude. + self.prelude = '' + self.prelude_length = 0 + self.postlude = '' + + self.task = None # SUBCLASS MUST DEFINE THIS + + def filter_tests(self, testcases): + """Return the relevant subset of the question's testcases. + This will be all testcases not marked precheck-only if it's not a precheck or all testcases if it is a + precheck and the question precheck is set to "All", or the appropriate subset in all other cases. + """ + if not self.params['IS_PRECHECK']: + return [test for test in testcases if test.testtype != TYPE_PRECHECKONLY] + elif self.params['QUESTION_PRECHECK'] == PRECHECK_ALL: + return testcases + elif self.params['QUESTION_PRECHECK'] == PRECHECK_EMPTY: + return [] + elif self.params['QUESTION_PRECHECK'] == PRECHECK_EXAMPLES: + return [test for test in testcases if test.useasexample] + elif self.params['QUESTION_PRECHECK'] == PRECHECK_SELECTED: + return [test for test in testcases if test.testtype in [TYPE_PRECHECKONLY, TYPE_BOTH]] + + def style_errors(self): + """Return a list of all the style errors. Implementation is language dependent. + Default is no style checking. + """ + return [] + + def single_program_build_possible(self): + """Return true if and only if the current configuration permits a single program to be + built and tried containing all tests. It should be true for write-a-program questions and + conditionally true for other types of questions that allow a "combinator" approach, + dependent on the presence of stdins in tests and other such conditions. + """ + raise NotImplementedError("Tester must have a single_program_build_possible method") + + def adjust_error_line_nums(self, error): + """Given a runtime error message, adjust it as may be required by the + language, e.g. adjusting line numbers + """ + raise NotImplementedError("Tester must have an adjust_error_line_nums method") + + def single_run_possible(self): + """Return true if a single program has been built and it is possible to use that in a single run + with all tests. + """ + return (self.task.executable_built + and not self.params['runtestssingly'] + and not self.result_table.has_stdins + and not self.params['testisbash']) + + def make_test_postlude(self, testcases): + """Return the postlude testing code containing all the testcode from + the given list of testcases (which may be the full set or a singleton list). + A separator must be printed between testcase outputs.""" + raise NotImplementedError("Tester must have a make_test_postlude method") + + def trial_compile(self): + """This function is the first check on the syntactic correctness of the submitted code. + It is called before any style checks are done. For compiled languages it should generally + call the standard language compiler on the student submitted code with any required prelude + added and, if possible, all tests included. CompileError should be raised if the compile fails, + which will abort all further testing. + If possible a complete ready-to-run executable should be built as well; if this succeeds, the + LanguageTasks 'executable_built' attribute should be set. This should be possible for write-a-program + questions or for write-a-function questions when there is no stdin data in any of the tests. + + Interpreted languages should perform what syntax checks are possible using the standard language tools. + If those checks succeeded, they should also attempt to construct a source program that incorporates all + the different tests (the old "combinator" approach) and ensure the task's 'executable_built' attribute + is True. + + The following implementation is sufficient for standard compiled languages like C, C++, Java. It + may need overriding for other languages. + """ + if self.single_program_build_possible(): + self.setup_for_test_runs(self.testcases) + make_executable = True + else: + self.postlude = '' + self.task.set_code(self.prelude + self.student_answer, self.prelude_length) + make_executable = False + + self.task.compile(make_executable) # Could raise CompileError + + def setup_for_test_runs(self, tests): + """Set the code and prelude length as appropriate for a run with all the given tests. May be called with + just a singleton list for tests if single_program_build_possible has returned false or if testing with + multiple tests has given exceptions. + This implementation may need to be overridden, e.g. if the student code should follow the test code, as + say in Matlab scripts. + """ + self.postlude = self.make_test_postlude(tests) + self.task.set_code(self.prelude + self.student_answer + self.postlude, self.prelude_length) + + def run_all_tests(self): + """Run all the tests, leaving self.ResultTable object containing all test results. + Can raise CompileError or RunError if things break. + If any runtime errors occur on the full test, drop back to running tests singly. + """ + done = False + if self.single_run_possible(): + # We have an executable ready to go, with no stdins or other show stoppers + output, error = self.task.run_code() + output = output.rstrip() + '\n' + error = error.strip() + '\n' + + # Generate a result table using all available test data. + results = output.split(self.separator + '\n') + errors = error.split(self.separator + '\n') + if len(results) == len(errors): + merged_results = [] + for result, error in zip(results, errors): + result = result.rstrip() + '\n' + if error.strip(): + adjusted_error = self.adjust_error_line_nums(error.rstrip()) + result += '\n*** RUN ERROR ***\n' + adjusted_error + merged_results.append(result) + + missed_tests = len(self.testcases) - len(merged_results) + + for test, output in zip(self.testcases, merged_results): + self.result_table.add_row(test, output) + + self.result_table.tests_missed(missed_tests) + if self.task.timed_out: + self.result_table.record_global_error(TIMEOUT_MESSAGE) + done = True + + if not done: + # Something broke. We will need to run each test case separately + self.task.executable_built = False + self.result_table.reset() + + if not done: + # If a single run isn't appropriate, do a separate run for each test case. + build_each_test = not self.task.executable_built + for i_test, test in enumerate(self.testcases): + if build_each_test: + self.setup_for_test_runs([test]) + self.task.compile(True) + standard_input = test.extra if self.params['stdinfromextra'] else test.stdin + if self.params['testisbash']: + output, error = self.task.run_code(standard_input, test.testcode) + else: + output, error = self.task.run_code(standard_input) + adjusted_error = self.adjust_error_line_nums(error.rstrip()) + self.result_table.add_row(test, output, adjusted_error) + if error and self.params['abortonerror']: + self.result_table.tests_missed(len(self.testcases) - i_test - 1) + break + + def compile_and_run(self): + """Phase one of the test operation: do a trial compile and then, if all is well and it's not a precheck, + continue on to run all tests. + Return a tuple mark, errors where mark is a fraction in 0 - 1 and errors is a list of all the errors. + self.test_results contains all the test details. + """ + mark = 0 + errors = [] + + # Do a trial compile, then a style check. If all is well, run the code + try: + self.trial_compile() + + if not self.params['nostylechecks']: + errors = self.style_errors() + if not errors: + if self.params['IS_PRECHECK'] and self.params['QUESTION_PRECHECK'] <= 1: + mark = 1 + else: + self.run_all_tests() + max_mark = sum(test.mark for test in self.testcases) + mark = self.result_table.get_mark() / max_mark # Fractional mark 0 - 1 + except languagetask.CompileError as err: + adjusted_error = self.adjust_error_line_nums(str(err).rstrip()) + errors.append("COMPILE ERROR\n" + adjusted_error) + except languagetask.RunError as err: + adjusted_error = self.adjust_error_line_nums(str(err).rstrip()) + errors.append('RUN ERROR\n' + adjusted_error) + return mark, errors + + def prerun_hook(self): + """A hook for subclasses to do initial setup or code hacks etc + Returns a list of errors, to which other errors are appended + """ + return [] + + def get_all_images_html(self): + r"""Search the current directory for images named _image.*(Expected|Got)(\d+).png. + For each such file construct an html img element with the data encoded + in a dataurl. + If we're running the sample answer, always return [] - images will be + picked up when we run the actual answer. + Returns a list of tuples (img_elements, column_name, row_number) where + column_name is either 'Expected' or 'Got', defining in which result table + column the image belongs and row number is the row (0-origin, excluding + the header row). + """ + images = [] + if self.params.get('running_sample_answer', False): + return [] + if self.params['imagewidth'] is not None: + width_spec = " width={}".format(self.params['imagewidth']) + else: + width_spec = "" + files = sorted(os.listdir('.')) + for filename in files: + match = re.match(r'_image[^.]*\.(Expected|Got)\.(\d+).png', filename) + if match: + image_data = get_jpeg_b64(filename) + img_template = '' + img_html = img_template.format(width_spec, image_data) + column = match.group(1) # Name of column + row = int(match.group(2)) # 0-origin row number + images.append((img_html, column, row)) + return images + + def test_code(self): + """The "main program" for testing. Returns the test outcome, ready to be printed by json.dumps""" + errors = self.prerun_hook() + if errors: + mark = 0 + else: + mark, errors = self.compile_and_run() + + outcome = {"fraction": mark} + + error_text = '\n'.join(errors) + # TODO - check if error line numbers are still being corrected in C and matlab + if self.params['IS_PRECHECK']: + if mark == 1: + prologue = "

Passed 🙂

" + else: + prologue = "

Failed, as follows.

" + elif errors: + prologue = "

Pre-run checks failed

\n" + else: + prologue = "" + + if prologue: + outcome['prologuehtml'] = prologue + self.htmlize(error_text) + + epilogue = '' + images = self.get_all_images_html() + if images: + for (image, column, row) in images: + self.result_table.add_image(image, column, row) + outcome['columnformats'] = self.result_table.get_column_formats() + + if len(self.result_table.table) > 1: + outcome['testresults'] = self.result_table.get_table() + outcome['showdifferences'] = True + + if self.result_table.global_error: + epilogue += "

Run Error

{}
".format( + self.htmlize(self.result_table.global_error)) + + if self.result_table.aborted: + epilogue = outcome.get('epiloguehtml', '') + ( + "
Testing was aborted due to runtime errors.
") + + if self.result_table.missing_tests != 0: + template = "
{} tests not run due to previous errors.
" + epilogue += template.format(self.result_table.missing_tests) + + if self.result_table.failed_hidden: + epilogue += "
One or more hidden tests failed.
" + + if epilogue: + outcome['epiloguehtml'] = epilogue + return outcome + + @staticmethod + def clean(code): + """Return the given code with trailing white space stripped from each line""" + return '\n'.join([line.rstrip() for line in code.split('\n')]) + '\n' + + @staticmethod + def htmlize(message): + """An html version of the given error message""" + return '
' + html.escape(message) + '
' if message else '' diff --git a/python3_scratchpad_plus/pytester.py b/python3_scratchpad_plus/pytester.py new file mode 100644 index 0000000..1819302 --- /dev/null +++ b/python3_scratchpad_plus/pytester.py @@ -0,0 +1,255 @@ +"""The main python-program testing class that does all the work - style checks, + run and grade. A subclass of the generic tester. + Since each test can by run within the current instance of Python using + an exec, we avoid the usual complication of combinators by running + each test separately regardless of presence of stdin, testcode, etc. +""" +import __pytask as pytask +import re +from __tester import Tester +from __pystylechecker import StyleChecker + + +class PyTester(Tester): + def __init__(self, params, testcases): + """Initialise the instance, given the test of template and global parameters plus + all the testcases. Parameters relevant to this class are all those listed for the Tester class plus + 'extra' which takes the values 'pretest' or 'posttest' (the other possible value, 'stdin', has been + handled by the main template). + Additionally the support classes like stylechecker and pyparser need their + own params - q.v. + """ + super().__init__(params, testcases) # Most of the task is handed by the generic tester + + # Py-dependent attributes + self.task = pytask.PyTask(params) + self.prelude = '' + + if params['isfunction']: + if not self.has_docstring(): + self.prelude = '"""Dummy docstring for a function"""\n' + + if params['usesmatplotlib']: + self.prelude += '\n'.join([ + 'import os', + 'import matplotlib as _mpl', + '_mpl.use("Agg")', + 'from __plottools import print_plot_info', + ]) + '\n' + self.params['pylintoptions'].append("--disable=ungrouped-imports") + + if params['usesnumpy']: + self.prelude += 'import numpy as np\n' + self.params['pylintoptions'].append("--disable=unused-import,ungrouped-imports") + self.params['pylintoptions'].append("--extension-pkg-whitelist=numpy") + + for import_string in params['imports']: + if ' ' not in import_string: + self.prelude += 'import ' + import_string + '\n' + else: + self.prelude += import_string + '\n' + + if params['prelude'] != '': + self.prelude += '\n' + params['prelude'].rstrip() + '\n' + + try: + with open('_prefix.py') as prefix: + prefix_code = prefix.read() + self.prelude += prefix_code.rstrip() + '\n' + + except FileNotFoundError: + pass + + self.prelude_length = len(self.prelude.splitlines()) + if self.has_docstring() and self.prelude_length > 0: + # If we insert prelude in front of the docstring, pylint will + # give a missing docstring error. Our horrible hack solution is + # to insert an extra docstring at the start and turn off the + # resulting 'string statement has no effect' error. + self.prelude = '"""Dummy docstring for a function"""\n' + self.prelude + self.prelude_length += 1 + self.params['pylintoptions'].append("--disable=W0105") + self.style_checker = StyleChecker(self.prelude, self.params['STUDENT_ANSWER'], self.params) + + def has_docstring(self): + """True if the student answer has a docstring, which means that, + when stripped, it starts with a string literal. + """ + prog = self.params['STUDENT_ANSWER'].lstrip() + return prog.startswith('"') or prog.startswith("'") + + def style_errors(self): + """Return a list of all the style errors. Start with local tests and continue with pylint + only if there are no local errors. + """ + errors = [] + if self.params.get('localprechecks', True): + try: + errors += self.style_checker.local_errors() # Note: prelude not included so don't adjust line nums + except Exception as e: + errors += [str(e)] + else: + check_for_passive = (self.params['warnifpassiveoutput'] and self.params['isfunction']) + if check_for_passive and self.passive_output(): + errors.append("Your code was not expected to generate any output " + + "when executed stand-alone.\nDid you accidentally include " + + "your test code?") + + if len(errors) == 0 or self.params.get('forcepylint', False): + # Run precheckers (pylint, mypy) + try: + # Style-check the program without any test cases or other postlude added + errors += self.style_checker.style_errors() + except Exception as e: + error_text = '*** Unexpected error while running precheckers. Please report ***\n' + str(e) + errors += [error_text] + errors = [self.simplify_error(self.adjust_error_line_nums(error)) for error in errors] + errors = [error for error in errors if not error.startswith('************* Module')] + + errors = [error.replace(', ', '') for error in errors] # Another error tidying operation + if errors: + errors.append("\nSorry, but your code doesn't pass the style checks.") + return errors + + def prerun_hook(self): + """A hook for subclasses to do initial setup or code hacks etc + Returns a list of errors, to which other errors are appended. + In this class we use it to perform required hacks to disable + calls to main. If the call to main_hacks fails, assume the code + is bad and will get flagged by pylint in due course. + """ + try: + return self.main_hacks() + except: + return [] + + def passive_output(self): + """ Return the passive output from the student answer code + This is essentially a "dry run" of the code. + """ + code = self.prelude + self.params['STUDENT_ANSWER'] + if self.params['usesmatplotlib']: + code += '\n'.join([ + 'figs = _mpl.pyplot.get_fignums()', + 'if figs:', + ' print(f"{len(figs)} figures found")' + ]) + '\n' + task = pytask.PyTask(self.params, code) + task.compile() + captured_output, captured_error = task.run_code() + return (captured_output + '\n' + captured_error).strip() + + def make_test_postlude(self, testcases): + """Return the code that follows the student answer containing all the testcode + from the given list of testcases, which should always be of length 1 + (because we don't bother trying to combine all the tests into a + single run in Python) + """ + assert len(testcases) == 1 + if self.params['notest']: + return '' + test = testcases[0] + tester = '' + if self.params['globalextra'] and self.params['globalextra'] == 'pretest': + tester += self.params['GLOBAL_EXTRA'] + '\n' + if test.extra and self.params['extra'] == 'pretest': + tester += test.extra + '\n' + if test.testcode: + tester += test.testcode.rstrip() + '\n' + if self.params['globalextra'] and self.params['globalextra'] == 'posttest': + tester += self.params['GLOBAL_EXTRA'] + '\n' + if test.extra and self.params['extra'] == 'posttest': + tester += test.extra + '\n' + + if self.params['usesmatplotlib']: + if 'dpi' in self.params and self.params['dpi']: + extra = f", dpi={self.params['dpi']}" + else: + extra = '' + if self.params.get('running_sample_answer', False): + column = 'Expected' + else: + column = 'Got' + test_num = len(self.result_table.table) - 1 # 0-origin test number from result table + tester += '\n'.join([ + 'figs = _mpl.pyplot.get_fignums()', + 'for fig in figs:', + ' _mpl.pyplot.figure(fig)', + ' row = {}'.format(test_num), + ' column = "{}"'.format(column), + ' _mpl.pyplot.savefig("_image{}.{}.{}.png".format(fig, column, row), bbox_inches="tight"' + '{})'.format(extra), + ' _mpl.pyplot.close(fig)' + ]) + '\n' + return tester + + def single_program_build_possible(self): + """We avoid all the complication of trying to run all tests in + a single subprocess run by using exec to run each test singly. + """ + return False + + def adjust_error_line_nums(self, error): + """Subtract the prelude length of all line numbers in the given error message + """ + error_patterns = [ + (r'(.*.* \(syntax-error\).*)', []), + (r'(.*File ".*", line +)(\d+)(, in .*)', [2]), + (r'(.*: *)(\d+)(, *\d+:.*\(.*line +)(\d+)(\).*)', [2, 4]), + (r'(.*: *)(\d+)(, *\d+:.*\(.*\).*)', [2]), + (r'(.*:)(\d+)(:\d+: [A-Z]\d+: .*line )(\d+)(.*)', [2, 4]), + (r'(.*:)(\d+)(:\d+: [A-Z]\d+: .*)', [2]), + ] + output_lines = [] + for line in error.splitlines(): + for pattern, line_group_nums in error_patterns: + match = re.match(pattern, line) + if match: + line = '' + for i, group in enumerate(match.groups(), 1): + if i in line_group_nums: + linenum = int(match.group(i)) + adjusted = linenum - self.prelude_length + line += str(adjusted) + else: + line += group + break + + output_lines.append(line) + return '\n'.join(output_lines) + + def simplify_error(self, error): + """Return a simplified version of a pylint error with Line inserted in + lieu of __source.py:

: Xnnnn + """ + pattern = r'__source.py:(\d+): *\d+: *[A-Z]\d+: (.*)' + match = re.match(pattern, error) + if match: + return f"Line {match.group(1)}: {match.group(2)}" + else: + return error + + def main_hacks(self): + """Modify the code to be tested if params stripmain or stripmainifpresent' + are specified. Returns a list of errors encountered while so doing. + """ + errors = [] + if self.params['stripmain'] or self.params['stripmainifpresent']: + main_calls = self.style_checker.find_function_calls('main') + if self.params['stripmain'] and main_calls == []: + errors.append("No call to main() found") + else: + student_lines = self.student_answer.split('\n') + for (line, depth) in main_calls: + if depth == 0: + main_call = student_lines[line] + if not re.match(r' *main\(\)', main_call): + errors.append(f"Illegal call to main().\n" + + "main should not take any parameters and should not return anything.") + else: + student_lines[line] = main_call.replace( + "main", "pass # Disabled call to main") + else: + student_lines[line] += " # We've let you call main here." + self.params['STUDENT_ANSWER'] = self.student_answer = '\n'.join(student_lines) + '\n' + + return errors diff --git a/python3_stage1/__pystylechecker (1).py b/python3_stage1/__pystylechecker (1).py new file mode 100644 index 0000000..2c296e5 --- /dev/null +++ b/python3_stage1/__pystylechecker (1).py @@ -0,0 +1,456 @@ +"""All the style checking code for Python3. + This special version disallows any global code + except function defs and global constants. +""" + +from io import BytesIO +import os +import subprocess +import ast +import tokenize +import token +import re +import shutil +from collections import defaultdict + +class StyleChecker: + def __init__(self, prelude, student_answer, params): + self.prelude = prelude + self.student_answer = student_answer + self.params = params + self.function_call_map = None + self._tree = None + + @property + def tree(self): + if self._tree is None: + self._tree = ast.parse(self.student_answer) + return self._tree + + def style_errors(self): + """Return a list of errors from local style checks plus pylint and/or mypy + """ + errors = [] + source = open('__source.py', 'w', encoding="utf-8") + code_to_check = self.prelude + self.student_answer + prelude_len = len(self.prelude.splitlines()) + source.write(code_to_check) + source.close() + env = os.environ.copy() + env['HOME'] = os.getcwd() + pylint_opts = self.params.get('pylintoptions',[]) + precheckers = self.params.get('precheckers', ['pylint']) + result = '' + + if 'pylint' in precheckers: + try: # Run pylint + cmd = 'python3.9 -m pylint ' + ' '.join(pylint_opts) + ' __source.py' + result = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env, + shell=True) + except Exception as e: + result = e.output + + else: + # (mct63) Abort if there are any comments containing 'pylint:'. + try: + tokenizer = tokenize.tokenize(BytesIO(self.student_answer.encode('utf-8')).readline) + for token_type, token_text, *_ in tokenizer: + if token_type == tokenize.COMMENT and 'pylint:' in token_text: + errors.append("Comments can not include 'pylint:'") + break + + except Exception: + errors.append("Something went wrong while parsing comments. Report this.") + + if "Using config file" in result: + result = '\n'.join(result.splitlines()[1:]).split() + + if result == '' and 'mypy' in precheckers: + code_to_check = 'from typing import List as list, Dict as dict, Tuple as tuple, Set as set, Any\n' + code_to_check + with open('__source2.py', 'w', encoding='utf-8') as outfile: + outfile.write(code_to_check) + cmd = 'python3.8 -m mypy --no-error-summary --no-strict-optional __source2.py' + try: # Run mypy + subprocess.check_output(cmd, # Raises an exception if there are errors + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env, + shell=True) + except Exception as e: + result = e.output + line_num_fix = lambda match: "Line " + str(int(match[1]) - 1 - prelude_len) + match[2] + result = re.sub(r'__source2.py:(\d+)(.*)', line_num_fix, result) + + if result == '' and self.params.get('requiretypehints', False): + bad_funcs = self.check_type_hints() + for fun in bad_funcs: + result += f"Function '{fun}' does not have correct type hints\n" + + if result: + errors = result.strip().splitlines() + errors.append("Sorry, but your code doesn't pass the style checks.") + + return errors + + def local_errors(self): + """Perform various local checks as specified by the current set of + template parameters. + """ + errors = [] + + for banned in self.params.get('proscribedsubstrings', []): + if banned in self.student_answer: + errors.append(f"The string '{banned}' is not permitted anywhere in your code.") + + if self.params.get('banglobalcode', True): + errors += self.find_global_code() + + if not self.params.get('allownestedfunctions', True): + # Except for legacy questions or where explicitly allowed, nested functions are banned + nested_funcs = self.find_nested_functions() + for func in nested_funcs: + errors.append("Function '{}' is defined inside another function".format(func)) + + max_length = self.params['maxfunctionlength'] + bad_funcs = self.find_too_long_funcs(max_length) + for func, count in bad_funcs: + errors.append("Function '{}' is too long\n({} statements, max is {})" + "".format(func, count, max_length)) + + bad_used = self.find_illegal_functions() + for name in bad_used: + errors.append("You called the banned function '{}'.".format(name)) + + missing_funcs = self.find_missing_required_function_calls() + for name in missing_funcs: + errors.append("You forgot to use the required function '{}'.".format(name)) + + missing_funcs = self.find_missing_required_function_definitions() + for name in missing_funcs: + errors.append("You forgot to define the required function '{}'.".format(name)) + + missing_constructs = self.find_missing_required_constructs() + for reqd in missing_constructs: + errors.append("Your program must include at least one " + reqd + " statement.") + + bad_constructs = self.find_illegal_constructs() + for notallowed in bad_constructs: + errors.append("Your program must not include any " + notallowed + "s.") + + num_constants = len([line for line in self.student_answer.split('\n') if re.match(' *[A-Z_][A-Z_0-9]* *=', line)]) + if num_constants > self.params['maxnumconstants']: + errors.append("You may not use more than " + str(self.params['maxnumconstants']) + " constants.") + + # (mct63) Check if anything restricted is being imported. + if 'restrictedmodules' in self.params: + restricted = self.params['restrictedmodules'] + for import_name, names in self.find_all_imports().items(): + if import_name in restricted: + if restricted[import_name].get('onlyallow', None) == []: + errors.append("Your program should not import anything from '{}'.".format(import_name)) + else: + for name in names: + if (('onlyallow' in restricted[import_name] and name not in restricted[import_name]['onlyallow']) or + name in restricted[import_name].get('disallow', [])): + errors.append("Your program should not import '{}' from '{}'.".format(name, import_name)) + + return errors + + def find_all_imports(self): + """Returns a dictionary mapping in which the keys are all modules + being imported and the values are a list of what things within + the module are being modules. An empty list indicates the entire + module is imported.""" + found_imports = {} + class ImportFinder(ast.NodeVisitor): + def visit_Import(self, node): + for alias in node.names: + if alias.name not in found_imports: + found_imports[alias.name] = [] + self.generic_visit(node) + def visit_ImportFrom(self, node): + if node.module not in found_imports: + found_imports[node.module] = [] + for alias in node.names: + found_imports[node.module].append(alias.name) + self.generic_visit(node) + + visitor = ImportFinder() + visitor.visit(self.tree) + return found_imports + + def find_all_function_calls(self): + """Return a dictionary mapping in which the keys are all functions + called by the source code and values are a list of + (line_number, nesting_depth) tuples.""" + class FuncFinder(ast.NodeVisitor): + + def __init__(self, *args, **kwargs): + self.depth = 0 + self.found_funcs = defaultdict(list) + super().__init__(*args, **kwargs) + + def visit_FunctionDef(self, node): + """ Every time we enter a function, we get 'deeper' into the code. + We want to note how deep a function is when we find its call.""" + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + + def visit_Call(self, node): + """A function has been called, so check its name + against the given one.""" + try: + if 'id' in dir(node.func): + name = node.func.id + else: + name = node.func.attr + # Line numbers are 1-indexed, so decrement by 1 + self.found_funcs[name].append((node.lineno - 1, self.depth)) + except AttributeError: + pass # either not calling a function (??) or it's not named. + self.generic_visit(node) + + if self.function_call_map is None: + visitor = FuncFinder() + visitor.visit(self.tree) + self.function_call_map = visitor.found_funcs + return self.function_call_map + + def find_defined_functions(self): + """Find all the functions defined.""" + defined = set() + class FuncFinder(ast.NodeVisitor): + + def __init__(self): + self.prefix = '' + + def visit_ClassDef(self, node): + old_prefix = self.prefix + self.prefix += node.name + '.' + self.generic_visit(node) + self.prefix = old_prefix + + def visit_FunctionDef(self, node): + defined.add(self.prefix + node.name) + old_prefix = self.prefix + self.prefix += node.name + '.' + self.generic_visit(node) + self.prefix = old_prefix + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + visitor = FuncFinder() + visitor.visit(self.tree) + return defined + + def constructs_used(self): + """Return a set of all constructs encountered in the parse tree""" + constructs_seen = set() + class ConstructFinder(ast.NodeVisitor): + def visit_Assert(self, node): + constructs_seen.add('assert') + self.generic_visit(node) + def visit_Raise(self, node): + constructs_seen.add('raise') + self.generic_visit(node) + def visit_Lambda(self, node): + constructs_seen.add('lambda') + self.generic_visit(node) + def visit_Import(self, node): + constructs_seen.add('import') + self.generic_visit(node) + def visit_ImportFrom(self, node): + constructs_seen.add('import') + self.generic_visit(node) + def visit_For(self, node): + constructs_seen.add('for') + self.generic_visit(node) + def visit_While(self, node): + constructs_seen.add('while') + self.generic_visit(node) + def visit_Comprehension(self, node): + constructs_seen.add('comprehension') + self.generic_visit(node) + def visit_ListComp(self, node): + constructs_seen.add('listcomprehension') + self.generic_visit(node) + def visit_SetComp(self, node): + constructs_seen.add('setcomprehension') + self.generic_visit(node) + def visit_DictComp(self, node): + constructs_seen.add('dictcomprehension') + self.generic_visit(node) + def visit_Slice(self, node): + constructs_seen.add('slice') + def visit_If(self, node): + constructs_seen.add('if') + self.generic_visit(node) + def visit_Break(self, node): + constructs_seen.add('break') + self.generic_visit(node) + def visit_Continue(self, node): + constructs_seen.add('continue') + self.generic_visit(node) + def visit_Try(self, node): + constructs_seen.add('try') + self.generic_visit(node) + def visit_TryExcept(self, node): + constructs_seen.add('try') + constructs_seen.add('except') + self.generic_visit(node) + def visit_TryFinally(self, node): + constructs_seen.add('try') + constructs_seen.add('finally') + self.generic_visit(node) + def visit_ExceptHandler(self, node): + constructs_seen.add('except') + self.generic_visit(node) + def visit_With(self, node): + constructs_seen.add('with') + self.generic_visit(node) + def visit_Yield(self, node): + constructs_seen.add('yield') + self.generic_visit(node) + def visit_YieldFrom(self, node): + constructs_seen.add('yield') + self.generic_visit(node) + def visit_Return(self, node): + constructs_seen.add('return') + self.generic_visit(node) + + visitor = ConstructFinder() + visitor.visit(self.tree) + return constructs_seen + + def check_type_hints(self): + """Return a list of the names of functions that don't have full type hinting.""" + unhinted = [] + class MyVisitor(ast.NodeVisitor): + def visit_FunctionDef(self, node): + if node.returns is None or any([arg.annotation is None for arg in node.args.args]): + unhinted.append(node.name) + + visitor = MyVisitor() + tree = self.tree + visitor.visit(self.tree) + return unhinted + + def find_function_calls(self, name): + """Look for occurances of a specific function call""" + return self.find_all_function_calls().get(name, []) + + def find_illegal_functions(self): + """Find a set of all the functions that the student uses + that they are not allowed to use. """ + func_calls = self.find_all_function_calls() + return func_calls.keys() & set(self.params['proscribedfunctions']) + + def find_missing_required_function_calls(self): + """Find a set of the required functions that the student fails to use""" + func_calls = self.find_all_function_calls() + return set(self.params['requiredfunctioncalls']) - func_calls.keys() + + def find_missing_required_function_definitions(self): + """Find a set of required functions that the student fails to define""" + func_defs = self.find_defined_functions() + return set(self.params['requiredfunctiondefinitions']) - func_defs + + def find_illegal_constructs(self): + """Find all the constructs that were used but not allowed""" + constructs = self.constructs_used() + return constructs & set(self.params['proscribedconstructs']) + + def find_missing_required_constructs(self): + """Find which of the required constructs were not used""" + constructs = self.constructs_used() + return set(self.params['requiredconstructs']) - constructs + + def find_too_long_funcs(self, max_length): + """Return a list of the functions that exceed the given max_length + Each list element is a tuple of the function name and the number of statements + in its body.""" + + bad_funcs = [] + + class MyVisitor(ast.NodeVisitor): + + def visit_FunctionDef(self, node): + + def count_statements(node): + """Number of statements in the given node and its children""" + count = 1 + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Str): + count = 0 + else: + for attr in ['body', 'orelse', 'finalbody']: + if hasattr(node, attr): + children = node.__dict__[attr] + count += sum(count_statements(child) for child in children) + return count + + num_statements = count_statements(node) - 1 # Disregard def itself + if num_statements > max_length: + bad_funcs.append((node.name, num_statements)) + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + visitor = MyVisitor() + visitor.visit(self.tree) + return bad_funcs + + def find_global_code(self): + """Return a list of error messages relating to the existence of + any global assignment, for, while and if nodes. Ignores + global assignment statements with an ALL_CAPS target.""" + + global_errors = [] + class MyVisitor(ast.NodeVisitor): + def visit_Assign(self, node): + if node.col_offset == 0: + if len(node.targets) > 1: + global_errors.append(f"Multiple targets in global assignment statement at line {node.lineno}") + elif not node.targets[0].id.isupper(): + global_errors.append(f"Global assignment statement at line {node.lineno}") + + def visit_For(self, node): + if node.col_offset == 0: + global_errors.append(f"Global for loop at line {node.lineno}") + + def visit_While(self, node): + if node.col_offset == 0: + global_errors.append(f"Global while loop at line {node.lineno}") + + def visit_If(self, node): + if node.col_offset == 0: + global_errors.append(f"Global if statement at line {node.lineno}") + + visitor = MyVisitor() + visitor.visit(self.tree) + return global_errors + + def find_nested_functions(self): + """Return a list of functions that are declared with non-global scope""" + bad_funcs = [] + + class MyVisitor(ast.NodeVisitor): + is_visiting_func = False + + def visit_FunctionDef(self, node): + if self.is_visiting_func: + bad_funcs.append(node.name) + self.is_visiting_func = True + self.generic_visit(node) # Visit all children recursively + self.is_visiting_func = False + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + visitor = MyVisitor() + visitor.visit(self.tree) + return bad_funcs diff --git a/python3_stage1/__pystylechecker.prog4fun.py b/python3_stage1/__pystylechecker.prog4fun.py new file mode 100644 index 0000000..acfe76f --- /dev/null +++ b/python3_stage1/__pystylechecker.prog4fun.py @@ -0,0 +1,491 @@ +"""All the style checking code for Python3""" + +from io import BytesIO +import os +import sys +import subprocess +import ast +import tokenize +import token +import re +import shutil +from collections import defaultdict + +class StyleChecker: + def __init__(self, prelude, student_answer, params): + self.prelude = prelude + self.student_answer = student_answer + self.params = params + self.function_call_map = None + self._tree = None + + @property + def tree(self): + if self._tree is None: + self._tree = ast.parse(self.student_answer) + return self._tree + + def style_errors(self): + """Return a list of errors from local style checks plus pylint and/or mypy + """ + errors = [] + source = open('__source.py', 'w', encoding="utf-8") + code_to_check = self.prelude + self.student_answer + prelude_len = len(self.prelude.splitlines()) + source.write(code_to_check) + source.close() + env = os.environ.copy() + env['HOME'] = os.getcwd() + pylint_opts = self.params.get('pylintoptions',[]) + precheckers = self.params.get('precheckers', ['pylint']) + result = '' + + if 'pylint' in precheckers: + try: # Run pylint + cmd = f'{sys.executable} -m pylint ' + ' '.join(pylint_opts) + ' __source.py' + result = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env, + shell=True) + except Exception as e: + result = e.output + + else: + # (mct63) Abort if there are any comments containing 'pylint:'. + try: + tokenizer = tokenize.tokenize(BytesIO(self.student_answer.encode('utf-8')).readline) + for token_type, token_text, *_ in tokenizer: + if token_type == tokenize.COMMENT and 'pylint:' in token_text: + errors.append("Comments can not include 'pylint:'") + break + + except Exception: + errors.append("Something went wrong while parsing comments. Report this.") + + if "Using config file" in result: + result = '\n'.join(result.splitlines()[1:]).split() + + if result == '' and 'mypy' in precheckers: + code_to_check = 'from typing import List as list, Dict as dict, Tuple as tuple, Set as set, Any\n' + code_to_check + with open('__source2.py', 'w', encoding='utf-8') as outfile: + outfile.write(code_to_check) + cmd = f'{sys.executable} -m mypy --no-error-summary --no-strict-optional __source2.py' + try: # Run mypy + subprocess.check_output(cmd, # Raises an exception if there are errors + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env, + shell=True) + except Exception as e: + result = e.output + line_num_fix = lambda match: "Line " + str(int(match[1]) - 1 - prelude_len) + match[2] + result = re.sub(r'__source2.py:(\d+)(.*)', line_num_fix, result) + + if result == '' and self.params.get('requiretypehints', False): + bad_funcs = self.check_type_hints() + for fun in bad_funcs: + result += f"Function '{fun}' does not have correct type hints\n" + + if result: + errors = result.strip().splitlines() + + return errors + + def prettied(self, construct): + """Expand, if possible, the name of the given Python construct to a more + user friendly version, e.g. 'listcomprehension' -> 'list comprehension' + """ + expanded = { + 'listcomprehension': 'list comprehension', + 'while': 'while loop', + 'for': 'for loop', + 'try': 'try ... except statement', + 'dictcomprehension': 'dictionary comprehension', + 'slice': 'slice' + } + if construct in expanded: + return expanded[construct] + else: + return f"{construct} statement" + + def local_errors(self): + """Perform various local checks as specified by the current set of + template parameters. + """ + errors = [] + + for banned in self.params.get('proscribedsubstrings', []): + if banned in self.student_answer: + errors.append(f"The string '{banned}' is not permitted anywhere in your code.") + + for required in self.params.get('requiredsubstrings', []): + if isinstance(required, str) and required not in self.student_answer: + errors.append(f'The string "{required}" must occur somewhere in your code.') + elif isinstance(required, dict): + if 'pattern' in required and not re.findall(required['pattern'], self.student_answer): + errors.append(required['errormessage']) + elif 'string' in required and required['string'] not in self.student_answer: + errors.append(required['errormessage']) + + if self.params.get('banglobalcode', True): + errors += self.find_global_code() + + if not self.params.get('allownestedfunctions', True): + # Except for legacy questions or where explicitly allowed, nested functions are banned + nested_funcs = self.find_nested_functions() + for func in nested_funcs: + errors.append("Function '{}' is defined inside another function".format(func)) + + max_length = self.params['maxfunctionlength'] + bad_funcs = self.find_too_long_funcs(max_length) + for func, count in bad_funcs: + errors.append("Function '{}' is too long\n({} statements, max is {})" + "".format(func, count, max_length)) + + bad_used = self.find_illegal_functions() + for name in bad_used: + errors.append("You called the banned function '{}'.".format(name)) + + missing_funcs = self.find_missing_required_function_calls() + for name in missing_funcs: + errors.append("You forgot to use the required function '{}'.".format(name)) + + missing_funcs = self.find_missing_required_function_definitions() + for name in missing_funcs: + errors.append("You forgot to define the required function '{}'.".format(name)) + + missing_constructs = self.find_missing_required_constructs() + for reqd in missing_constructs: + expanded = self.prettied(reqd) + errors.append(f"Your program must include at least one {expanded}.") + + bad_constructs = self.find_illegal_constructs() + for notallowed in bad_constructs: + expanded = self.prettied(notallowed) + errors.append(f"Your program must not include any {expanded}s.") + + num_constants = len([line for line in self.student_answer.split('\n') if re.match(' *[A-Z_][A-Z_0-9]* *=', line)]) + if num_constants > self.params['maxnumconstants']: + errors.append("You may not use more than " + str(self.params['maxnumconstants']) + " constants.") + + # (mct63) Check if anything restricted is being imported. + if 'restrictedmodules' in self.params: + restricted = self.params['restrictedmodules'] + for import_name, names in self.find_all_imports().items(): + if import_name in restricted: + if restricted[import_name].get('onlyallow', None) == []: + errors.append("Your program should not import anything from '{}'.".format(import_name)) + else: + for name in names: + if (('onlyallow' in restricted[import_name] and name not in restricted[import_name]['onlyallow']) or + name in restricted[import_name].get('disallow', [])): + errors.append("Your program should not import '{}' from '{}'.".format(name, import_name)) + + return errors + + def find_all_imports(self): + """Returns a dictionary mapping in which the keys are all modules + being imported and the values are a list of what things within + the module are being modules. An empty list indicates the entire + module is imported.""" + found_imports = {} + class ImportFinder(ast.NodeVisitor): + def visit_Import(self, node): + for alias in node.names: + if alias.name not in found_imports: + found_imports[alias.name] = [] + self.generic_visit(node) + def visit_ImportFrom(self, node): + if node.module not in found_imports: + found_imports[node.module] = [] + for alias in node.names: + found_imports[node.module].append(alias.name) + self.generic_visit(node) + + visitor = ImportFinder() + visitor.visit(self.tree) + return found_imports + + def find_all_function_calls(self): + """Return a dictionary mapping in which the keys are all functions + called by the source code and values are a list of + (line_number, nesting_depth) tuples.""" + class FuncFinder(ast.NodeVisitor): + + def __init__(self, *args, **kwargs): + self.depth = 0 + self.found_funcs = defaultdict(list) + super().__init__(*args, **kwargs) + + def visit_FunctionDef(self, node): + """ Every time we enter a function, we get 'deeper' into the code. + We want to note how deep a function is when we find its call.""" + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + + def visit_Call(self, node): + """A function has been called, so check its name + against the given one.""" + try: + if 'id' in dir(node.func): + name = node.func.id + else: + name = node.func.attr + # Line numbers are 1-indexed, so decrement by 1 + self.found_funcs[name].append((node.lineno - 1, self.depth)) + except AttributeError: + pass # either not calling a function (??) or it's not named. + self.generic_visit(node) + + if self.function_call_map is None: + visitor = FuncFinder() + visitor.visit(self.tree) + self.function_call_map = visitor.found_funcs + return self.function_call_map + + + def find_defined_functions(self): + """Find all the functions defined.""" + defined = set() + class FuncFinder(ast.NodeVisitor): + + def __init__(self): + self.prefix = '' + + def visit_ClassDef(self, node): + old_prefix = self.prefix + self.prefix += node.name + '.' + self.generic_visit(node) + self.prefix = old_prefix + + def visit_FunctionDef(self, node): + defined.add(self.prefix + node.name) + old_prefix = self.prefix + self.prefix += node.name + '.' + self.generic_visit(node) + self.prefix = old_prefix + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + visitor = FuncFinder() + visitor.visit(self.tree) + return defined + + + def constructs_used(self): + """Return a set of all constructs encountered in the parse tree""" + constructs_seen = set() + class ConstructFinder(ast.NodeVisitor): + def visit_Assert(self, node): + constructs_seen.add('assert') + self.generic_visit(node) + def visit_Raise(self, node): + constructs_seen.add('raise') + self.generic_visit(node) + def visit_Lambda(self, node): + constructs_seen.add('lambda') + self.generic_visit(node) + def visit_Import(self, node): + constructs_seen.add('import') + self.generic_visit(node) + def visit_ImportFrom(self, node): + constructs_seen.add('import') + self.generic_visit(node) + def visit_For(self, node): + constructs_seen.add('for') + self.generic_visit(node) + def visit_While(self, node): + constructs_seen.add('while') + self.generic_visit(node) + def visit_Comprehension(self, node): + constructs_seen.add('comprehension') + self.generic_visit(node) + def visit_ListComp(self, node): + constructs_seen.add('listcomprehension') + self.generic_visit(node) + def visit_SetComp(self, node): + constructs_seen.add('setcomprehension') + self.generic_visit(node) + def visit_DictComp(self, node): + constructs_seen.add('dictcomprehension') + self.generic_visit(node) + def visit_Slice(self, node): + constructs_seen.add('slice') + def visit_If(self, node): + constructs_seen.add('if') + self.generic_visit(node) + def visit_Break(self, node): + constructs_seen.add('break') + self.generic_visit(node) + def visit_Continue(self, node): + constructs_seen.add('continue') + self.generic_visit(node) + def visit_Try(self, node): + constructs_seen.add('try') + self.generic_visit(node) + def visit_TryExcept(self, node): + constructs_seen.add('try') + constructs_seen.add('except') + self.generic_visit(node) + def visit_TryFinally(self, node): + constructs_seen.add('try') + constructs_seen.add('finally') + self.generic_visit(node) + def visit_ExceptHandler(self, node): + constructs_seen.add('except') + self.generic_visit(node) + def visit_With(self, node): + constructs_seen.add('with') + self.generic_visit(node) + def visit_Yield(self, node): + constructs_seen.add('yield') + self.generic_visit(node) + def visit_YieldFrom(self, node): + constructs_seen.add('yield') + self.generic_visit(node) + def visit_Return(self, node): + constructs_seen.add('return') + self.generic_visit(node) + + visitor = ConstructFinder() + visitor.visit(self.tree) + return constructs_seen + + def check_type_hints(self): + """Return a list of the names of functions that don't have full type hinting.""" + unhinted = [] + class MyVisitor(ast.NodeVisitor): + def visit_FunctionDef(self, node): + if node.returns is None or any([arg.annotation is None for arg in node.args.args]): + unhinted.append(node.name) + + visitor = MyVisitor() + tree = self.tree + visitor.visit(self.tree) + return unhinted + + def find_function_calls(self, name): + """Look for occurances of a specific function call""" + return self.find_all_function_calls().get(name, []) + + + def find_illegal_functions(self): + """Find a set of all the functions that the student uses + that they are not allowed to use. """ + func_calls = self.find_all_function_calls() + return func_calls.keys() & set(self.params['proscribedfunctions']) + + + def find_missing_required_function_calls(self): + """Find a set of the required functions that the student fails to use""" + func_calls = self.find_all_function_calls() + return set(self.params['requiredfunctioncalls']) - func_calls.keys() + + + def find_missing_required_function_definitions(self): + """Find a set of required functions that the student fails to define""" + func_defs = self.find_defined_functions() + return set(self.params['requiredfunctiondefinitions']) - func_defs + + + def find_illegal_constructs(self): + """Find all the constructs that were used but not allowed""" + constructs = self.constructs_used() + return constructs & set(self.params['proscribedconstructs']) + + + def find_missing_required_constructs(self): + """Find which of the required constructs were not used""" + constructs = self.constructs_used() + return set(self.params['requiredconstructs']) - constructs + + + def find_too_long_funcs(self, max_length): + """Return a list of the functions that exceed the given max_length + Each list element is a tuple of the function name and the number of statements + in its body.""" + + bad_funcs = [] + + class MyVisitor(ast.NodeVisitor): + + def visit_FunctionDef(self, node): + + def count_statements(node): + """Number of statements in the given node and its children""" + count = 1 + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Str): + count = 0 + else: + for attr in ['body', 'orelse', 'finalbody']: + if hasattr(node, attr): + children = node.__dict__[attr] + count += sum(count_statements(child) for child in children) + return count + + num_statements = count_statements(node) - 1 # Disregard def itself + if num_statements > max_length: + bad_funcs.append((node.name, num_statements)) + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + visitor = MyVisitor() + visitor.visit(self.tree) + return bad_funcs + + + def find_global_code(self): + """Return a list of error messages relating to the existence of + any global assignment, for, while and if nodes. Ignores + global assignment statements with an ALL_CAPS target.""" + + global_errors = [] + class MyVisitor(ast.NodeVisitor): + def visit_Assign(self, node): + if node.col_offset == 0: + if len(node.targets) > 1 or isinstance(node.targets[0], ast.Tuple): + global_errors.append(f"Multiple targets in global assignment statement at line {node.lineno}") + elif not node.targets[0].id.isupper(): + global_errors.append(f"Global assignment statement at line {node.lineno}") + + def visit_For(self, node): + if node.col_offset == 0: + global_errors.append(f"Global for loop at line {node.lineno}") + + def visit_While(self, node): + if node.col_offset == 0: + global_errors.append(f"Global while loop at line {node.lineno}") + + def visit_If(self, node): + if node.col_offset == 0: + global_errors.append(f"Global if statement at line {node.lineno}") + + visitor = MyVisitor() + visitor.visit(self.tree) + return global_errors + + + def find_nested_functions(self): + """Return a list of functions that are declared with non-global scope""" + bad_funcs = [] + + class MyVisitor(ast.NodeVisitor): + is_visiting_func = False + + def visit_FunctionDef(self, node): + if self.is_visiting_func: + bad_funcs.append(node.name) + self.is_visiting_func = True + self.generic_visit(node) # Visit all children recursively + self.is_visiting_func = False + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + visitor = MyVisitor() + visitor.visit(self.tree) + return bad_funcs diff --git a/python3_stage1_gapfiller/__tester.py b/python3_stage1_gapfiller/__tester.py index 82e98f1..78a1dd2 100644 --- a/python3_stage1_gapfiller/__tester.py +++ b/python3_stage1_gapfiller/__tester.py @@ -252,7 +252,7 @@ def prerun_hook(self): return [] def get_all_images_html(self): - """Search the current directory for images named _image.*(Expected|Got)(\d+).png. + r"""Search the current directory for images named _image.*(Expected|Got)(\d+).png. For each such file construct an html img element with the data encoded in a dataurl. If we're running the sample answer, always return [] - images will be diff --git a/python3_stage1_gapfiller/pytester.py b/python3_stage1_gapfiller/pytester.py index 7b169d6..1819302 100644 --- a/python3_stage1_gapfiller/pytester.py +++ b/python3_stage1_gapfiller/pytester.py @@ -79,28 +79,36 @@ def has_docstring(self): return prog.startswith('"') or prog.startswith("'") def style_errors(self): - """Return a list of all the style errors.""" - try: - # Style-check the program without any test cases or other postlude added - errors = self.style_checker.style_errors() - except Exception as e: - error_text = '*** Unexpected error while runner precheckers. Please report ***\n' + str(e) - errors = [error_text] - errors = [self.adjust_error_line_nums(error) for error in errors] - - if len(errors) == 0: + """Return a list of all the style errors. Start with local tests and continue with pylint + only if there are no local errors. + """ + errors = [] + if self.params.get('localprechecks', True): try: - errors = self.style_checker.local_errors() # Note: prelude not included so don't adjust line nums + errors += self.style_checker.local_errors() # Note: prelude not included so don't adjust line nums except Exception as e: - error_text = '*** Unexpected error while doing local style checks. Please report ***\n' + str(e) - errors = [error_text] - - check_for_passive = (self.params['warnifpassiveoutput'] and self.params['isfunction']) - if len(errors) == 0 and check_for_passive and self.passive_output(): - errors.append("Your code was not expected to generate any output " + - "when executed stand-alone.\nDid you accidentally include " + - "your test code?") - + errors += [str(e)] + else: + check_for_passive = (self.params['warnifpassiveoutput'] and self.params['isfunction']) + if check_for_passive and self.passive_output(): + errors.append("Your code was not expected to generate any output " + + "when executed stand-alone.\nDid you accidentally include " + + "your test code?") + + if len(errors) == 0 or self.params.get('forcepylint', False): + # Run precheckers (pylint, mypy) + try: + # Style-check the program without any test cases or other postlude added + errors += self.style_checker.style_errors() + except Exception as e: + error_text = '*** Unexpected error while running precheckers. Please report ***\n' + str(e) + errors += [error_text] + errors = [self.simplify_error(self.adjust_error_line_nums(error)) for error in errors] + errors = [error for error in errors if not error.startswith('************* Module')] + + errors = [error.replace(', ', '') for error in errors] # Another error tidying operation + if errors: + errors.append("\nSorry, but your code doesn't pass the style checks.") return errors def prerun_hook(self): @@ -154,6 +162,10 @@ def make_test_postlude(self, testcases): tester += test.extra + '\n' if self.params['usesmatplotlib']: + if 'dpi' in self.params and self.params['dpi']: + extra = f", dpi={self.params['dpi']}" + else: + extra = '' if self.params.get('running_sample_answer', False): column = 'Expected' else: @@ -165,7 +177,7 @@ def make_test_postlude(self, testcases): ' _mpl.pyplot.figure(fig)', ' row = {}'.format(test_num), ' column = "{}"'.format(column), - ' _mpl.pyplot.savefig("_image{}.{}.{}.png".format(fig, column, row), bbox_inches="tight")', + ' _mpl.pyplot.savefig("_image{}.{}.{}.png".format(fig, column, row), bbox_inches="tight"' + '{})'.format(extra), ' _mpl.pyplot.close(fig)' ]) + '\n' return tester @@ -205,6 +217,17 @@ def adjust_error_line_nums(self, error): output_lines.append(line) return '\n'.join(output_lines) + def simplify_error(self, error): + """Return a simplified version of a pylint error with Line inserted in + lieu of __source.py:

: Xnnnn + """ + pattern = r'__source.py:(\d+): *\d+: *[A-Z]\d+: (.*)' + match = re.match(pattern, error) + if match: + return f"Line {match.group(1)}: {match.group(2)}" + else: + return error + def main_hacks(self): """Modify the code to be tested if params stripmain or stripmainifpresent' are specified. Returns a list of errors encountered while so doing. @@ -219,7 +242,7 @@ def main_hacks(self): for (line, depth) in main_calls: if depth == 0: main_call = student_lines[line] - if not re.match(' *main\(\)', main_call): + if not re.match(r' *main\(\)', main_call): errors.append(f"Illegal call to main().\n" + "main should not take any parameters and should not return anything.") else: diff --git a/python3_state_sequence_table/PROTOTYPE_python3_state_sequence_table_tester.xml b/python3_state_sequence_table/PROTOTYPE_python3_state_sequence_table_tester.xml new file mode 100644 index 0000000..6e3d406 --- /dev/null +++ b/python3_state_sequence_table/PROTOTYPE_python3_state_sequence_table_tester.xml @@ -0,0 +1,631 @@ + + + + + + PROTOTYPE_python3_state_sequence_table_tester + + + The prototype for a question type that asks a student to step through the execution of a given bit of code line-by-line, updating the program state at each step in a given table.

+

This is a skeleton rewrite of the SST question type in which the SST parameters that were previously set by template parameters are now set by a control panel within the question. Only the num_vars and num_rows parameters are implemented so far. All the others (input_width, output_width, return_width) are still to be done. And the control panel should be hidden from the student, visible only to the teacher.

+

To create a new question of this type:

+
    +
  1. Create a new coderunner question of type python3_state_sequence_table_tester.
    2. +
  2. Fill in the question name and question text and set the output for the first test case to OK.
    3. +
  3. Save.
    4. +
  4. Open the Answer Preload section.
    5. +
  5. Set the parameters in the answer to whatever you want.
    6. +
  6. Fill in the answer SST with the required values.
    7. +
  7. Save
    8. +
+

]]> + + + + + 1 + 0 + 0 + + python3_state_sequence_table_tester + 2 + 1 + 10, 20, ... + 0 + 0 + 0 + 10 + + + + 1 + + + 0 + 0 + + 0 + #\n|ms]]> + python3 + + + EqualityGrader + + + + + 1 + 1 + None + 0 + {} + 1 + html + + 0 + 0 + 10240 + + + 1 + 0 + +

SST parameters

+ num_vars: +
+ num_rows: +
+ input_width: +
+ output_width: +
+ return_width: +
+

+
+
+ + ]]> + + + + + + + + + + OK + + + + + + SHOW + + + + + + \ No newline at end of file diff --git a/python3_state_sequence_table/myoriginal_take2_prototype_extra.html b/python3_state_sequence_table/myoriginal_take2_prototype_extra.html new file mode 100644 index 0000000..04cf560 --- /dev/null +++ b/python3_state_sequence_table/myoriginal_take2_prototype_extra.html @@ -0,0 +1,424 @@ + + + +
+
+ + \ No newline at end of file diff --git a/python3_state_sequence_table/myoriginalprototype_extra.html b/python3_state_sequence_table/myoriginalprototype_extra.html new file mode 100644 index 0000000..4aea39d --- /dev/null +++ b/python3_state_sequence_table/myoriginalprototype_extra.html @@ -0,0 +1,357 @@ +
+

SST parameters (should be hidden from user)

+ num_vars: +
+ num_rows: + +
+ +

The SST itself

+
+
+ + \ No newline at end of file diff --git a/python3_state_sequence_table/prototype_extra.html b/python3_state_sequence_table/prototype_extra.html new file mode 100644 index 0000000..1615725 --- /dev/null +++ b/python3_state_sequence_table/prototype_extra.html @@ -0,0 +1,428 @@ +
+

SST parameters

+ + + + + + + +
VariableValue
num_vars
num_rows
var_widths
output_width
return_width
+
+

+
+
+ + \ No newline at end of file diff --git a/python3_test_table/prototypeextra.html b/python3_test_table/prototypeextra.html new file mode 100644 index 0000000..e963b95 --- /dev/null +++ b/python3_test_table/prototypeextra.html @@ -0,0 +1,317 @@ + + + + +
+ + + +
Test table
+
+
+
+ + + + + + \ No newline at end of file diff --git a/python3_test_table/template.py b/python3_test_table/template.py new file mode 100644 index 0000000..f84f153 --- /dev/null +++ b/python3_test_table/template.py @@ -0,0 +1,360 @@ +import locale +import json +import os +import re +import html +import random + +from pytester import PyTester + +STANDARD_PYLINT_OPTIONS = ['--disable=trailing-whitespace,superfluous-parens,' + + 'too-few-public-methods,consider-using-f-string,' + + 'unbalanced-tuple-unpacking,too-many-statements,' + + 'consider-using-enumerate,simplifiable-if-statement,' + + 'consider-iterating-dictionary,trailing-newlines,no-else-return,' + + 'consider-using-dict-comprehension,consider-using-generator,' + + 'len-as-condition,inconsistent-return-statements,consider-using-join,' + + 'singleton-comparison,unused-variable,chained-comparison,no-else-break,' + + 'consider-using-in,useless-object-inheritance,unnecessary-pass,' + + 'reimported,wrong-import-order,wrong-import-position,ungrouped-imports,' + + 'consider-using-set-comprehension,no-else-raise,' + + 'unspecified-encoding,use-dict-literal,,consider-using-with,' + + 'duplicate-string-formatting-argument,consider-using-dict-items,' + + 'consider-using-max-builtin,use-a-generator ', + '--good-names=i,j,k,n,s,c,_' + ] + + +locale.setlocale(locale.LC_ALL, 'C.UTF-8') + +KNOWN_PARAMS = { + 'abortonerror': True, + 'allowglobals': False, + 'banglobalcode': True, + 'allownestedfunctions': False, + 'checktemplateparams': True, + 'dpi': 65, + 'echostandardinput': True, + 'extra': 'None', + 'floattolerance': None, + 'forcepylint': False, + 'globalextra': 'None', + 'imagewidth': None, + 'imports': [], + 'isfunction': True, + 'localprechecks': True, + 'maxfunctionlength': 30, + 'maxnumconstants': 4, + 'maxoutputbytes': 10000, + 'maxstringlength': 2000, + 'norun': False, + 'nostylechecks': False, + 'notest': False, + 'parsonsproblemthreshold': None, # The number of checks before parsons' problem displayed + 'precheckers': ['pylint'], + 'prelude': '', + 'proscribedbuiltins': ['exec', 'eval'], + 'proscribedfunctions': [], + 'proscribedconstructs': ["goto"], + 'proscribedsubstrings': [], + 'pylintoptions': [], + 'requiredconstructs': [], + 'requiredfunctiondefinitions': [], + 'requiredfunctioncalls': [], + 'requiredsubstrings': [], + 'requiretypehints': False, + 'restrictedfiles': { + 'disallow': ['__.*', 'prog.*', 'pytester.py'], + }, + 'restrictedmodules': { + 'builtins': { + 'onlyallow': [] + }, + 'imp': { + 'onlyallow': [] + }, + 'importlib': { + 'onlyallow': [] + }, + 'os': { + 'disallow': ['system', '_exit', '_.*'] + }, + 'subprocess': { + 'onlyallow': [] + }, + 'sys': { + 'disallow': ['_.*'] + }, + }, + 'runextra': False, + 'showfeedbackwhenright': False, + 'stdinfromextra': False, + 'strictwhitespace': True, + 'stripmain': False, + 'stripmainifpresent': False, + 'testisbash': False, + 'testre': '', + 'timeout': 5, + 'totaltimeout': 50, + 'suppresspassiveoutput': False, + 'useanswerfortests': False, + 'usesmatplotlib': False, + 'usesnumpy': False, + 'usesubprocess': False, + 'warnifpassiveoutput': True, +} + +class TestCase: + def __init__(self, dict_rep): + """Construct a testcase from a dictionary representation obtained via JSON""" + self.testcode = dict_rep['testcode'] + self.stdin = dict_rep['stdin'] + self.expected = dict_rep['expected'] + self.extra = dict_rep['extra'] + self.display = dict_rep['display'] + try: + self.testtype = int(dict_rep['testtype']) + except: + self.testtype = 0 + self.hiderestiffail = bool(int(dict_rep['hiderestiffail'])) + self.useasexample = bool(int(dict_rep['useasexample'])) + self.mark = float(dict_rep['mark']) + + +# ================= CODE TO DO ALL TWIG PARAMETER PROCESSING =================== + +def process_template_params(): + """Extract the template params into a global dictionary PARAMS""" + global PARAMS + PARAMS = json.loads("""{{ QUESTION.parameters | json_encode | e('py') }}""") + checktemplateparams = PARAMS.get('checktemplateparams', True) + if checktemplateparams: + unknown_params = set(PARAMS.keys()) - set(KNOWN_PARAMS.keys()) + filtered_params = [param for param in unknown_params if not param.startswith('_')] + if filtered_params: + print("Unexpected template parameter(s):", list(sorted(filtered_params))) + + for param_name, default in KNOWN_PARAMS.items(): + if param_name in PARAMS: + param = PARAMS[param_name] + if type(param) != type(default) and default is not None: + print("Template parameter {} has wrong type (expected {})".format(param_name, type(default))) + else: + PARAMS[param_name] = default; + + if PARAMS['extra'] == 'stdin': + PARAMS['stdinfromextra'] = True + if PARAMS['runextra']: + PARAMS['extra'] = 'pretest' # Legacy support + if PARAMS['timeout'] < 2: + PARAMS['timeout'] = 2 # Allow 1 extra second freeboard + PARAMS['pylintoptions'] = STANDARD_PYLINT_OPTIONS + PARAMS['pylintoptions'] + if PARAMS['allowglobals']: + PARAMS['pylintoptions'].append("--const-rgx='[a-zA-Z_][a-zA-Z0-9_]{2,30}$'") + if PARAMS['usesmatplotlib']: + PARAMS['pylintoptions'].append("--disable=reimported,wrong-import-position,wrong-import-order,unused-import") + if PARAMS['testisbash']: + print("testisbash is not implemented for Python") + + +def get_test_cases(): + """Return an array of Test objects from the template parameter TESTCASES""" + test_cases = [TestCase(test) for test in json.loads("""{{ TESTCASES | json_encode | e('py') }}""")] + return test_cases + + +def get_tests_from_example_table(): + """Return a list of Test objects build from the Example table for this question type""" + num_rows = len(PARAMS['EXAMPLE_TESTS']) + tests = [] + for i in range(num_rows): + testcode = PARAMS['EXAMPLE_TESTS'][i] + result = PARAMS['EXAMPLE_EXPECTEDS'][i] + test = TestCase({ + 'testcode': testcode, + 'stdin': '', + 'expected': result, + 'extra': '', + 'display': 'show', + 'hiderestiffail': False, + 'useasexample': True, + 'mark': 1.0 + }) + tests.append(test) + return tests + + +def build_new_outcome(outcome): + """Replace the testresults table with customised prologuehtml that + contains a table more appropriate to this question type. + """ + GREYS = ['#f0f0f0', '#e0e0e0'] + IS_CORRECT_COL = 4 + + def style(colour, padding=5, border='border:1px solid darkgray', extrastyle=''): + """The style to use for a cell""" + return f'style="background-color:{colour};padding:{padding}px;{border};{extrastyle};"' + + locked = get_readonly_cells() + html = '\n' + html += f'' + for i, row in enumerate(outcome['testresults'][1:]): + is_correct = row[IS_CORRECT_COL] + html += '' + + # Extract the testcode and expected from the result table. + # Colour the cells according to whether they were user-editable and + # if so according to whether the test was labelled correct or not. + for result_table_col, test_table_col in [(1, 0), (2, 1)]: + row_colour = colour = GREYS[(i + 1) % 2] + if (i, test_table_col) not in locked and not is_correct: + # If this is a cell allowing user entry and test is wrong + colour = '#fcc' # Red + html += f'' + if is_correct: + html += f'\n' + else: + html += f'\n' + + html += '
TestResultValid test?
{row[result_table_col]}
✔✘
\n' + outcome['epiloguehtml'] = html + del outcome['testresults'] + return outcome + + +def get_answer(): + """Return the sample answer""" + answer_json = """{{QUESTION.answer | e('py')}}""".strip() + try: + answer = json.loads(answer_json)['main_answer_code'][0] + except: + answer = answer_json # Assume this is the original solution + return answer + + +def get_readonly_cells(): + """Return a list of the readonly cells""" + preload = json.loads("""{{QUESTION.answerpreload | e('py')}}""".strip()) + cells = [] + for cell_label in json.loads(preload['cr_readonly_cells'][0]).keys(): + mat = re.match(r'cell-(\d+)-(\d+)', cell_label) + cells.append((int(mat[1]), int(mat[2]))) # (row, column) + return cells + + +def process_global_params(): + """Plug into the PARAMS variable all the "global" parameters from + the question and its answer (as distinct from the template parameters). + """ + response = json.loads("""{{ STUDENT_ANSWER | e('py') }}""") + PARAMS['STUDENT_ANSWER'] = response['main_answer_code'][0].rstrip() + '\n' + PARAMS['EXAMPLE_TESTS'] = response['test_table_col0'] + PARAMS['EXAMPLE_EXPECTEDS'] = response['test_table_col1'] + PARAMS['SEPARATOR'] = "##" + PARAMS['IS_PRECHECK'] = "{{ IS_PRECHECK }}" == "1" + PARAMS['QUESTION_PRECHECK'] = {{ QUESTION.precheck }} # Type of precheck: 0 = None, 1 = Empty etc + PARAMS['ALL_OR_NOTHING'] = "{{ QUESTION.allornothing }}" == "1" # Whether or not all-or-nothing grading is being used + PARAMS['GLOBAL_EXTRA'] = """{{ QUESTION.globalextra | e('py') }}\n""" + PARAMS['STEP_INFO'] = json.loads("""{{ QUESTION.stepinfo | json_encode }}""") + answer = get_answer() + if answer: + if PARAMS['STUDENT_ANSWER'].strip() == answer.strip(): + PARAMS['AUTHOR_ANSWER'] = "

Your answer is an exact match with the author's solution.

" + else: + with open("__author_solution.html") as file: + PARAMS['AUTHOR_ANSWER'] = (file.read().strip() % html.escape(answer)) + else: + PARAMS['AUTHOR_ANSWER'] = PARAMS['AUTHOR_ANSWER_SCRAMBLED'] = '' + + +def update_test_cases(test_cases, outcome): + """Return the updated testcases after replacing all empty expected fields with those from the + given outcome's test_results which must have a column header 'Got'. Non-empty existing expected + fields are left unchanged. + If any errors occur, the return value will be None and the outcome parameter will have had its prologuehtml + value updated to include an error message. + """ + try: + results = outcome['testresults'] + col_num = results[0].index('Got') + for i in range(len(test_cases)): + if test_cases[i].expected.strip() == '': + test_cases[i].expected = results[i + 1][col_num] + except ValueError: + outcome['prologuehtml'] = "No 'Got' column in result table from which to get testcase expecteds" + test_cases = None + except Exception as e: + outcome['prologuehtml'] = "Unexpected error ({}) extracting testcase expecteds from sample answer output".format(e) + test_cases = None + return test_cases + + +def run_tests(params, test_cases): + """Run all tests using the sample answer rather than the student answer""" + new_params = {key: value for key, value in params.items()} + new_params['IS_PRECHECK'] = False + new_params['nostylechecks'] = True + new_params['STUDENT_ANSWER'] = get_answer() + new_params['running_sample_answer'] = True + tester = PyTester(new_params, test_cases) + outcome = tester.test_code() + return outcome + + +def get_expecteds_from_answer(params, test_cases): + """Run all tests using the sample answer rather than the student answer. + Fill in the expected field of each test case using the sample answer and return + the updated test case list. + Return None if the sample answer gave any sort of runtime error + """ + outcome = run_tests(params, test_cases) + if 'prologuehtml' in outcome: + outcome['prologuehtml'] = "

ERROR IN QUESTION'S SAMPLE ANSWER. PLEASE REPORT

\n" + outcome['prologuehtml'] + return outcome, None + else: + return outcome, update_test_cases(test_cases, outcome) + + +ok = True +process_template_params() +test_cases = get_test_cases() +process_global_params() +example_tests = get_tests_from_example_table() + +outcome = {'fraction': 1.0} + +if PARAMS['testre']: + # Check all tests to see if they contain the desired regular expression + bad_tests = [] + for test in example_tests: + code = test.testcode + if not re.search(PARAMS['testre'], code, re.DOTALL): + bad_tests.append(code) + if bad_tests: + test_is = 'test is' if len(bad_tests) == 1 else 'tests are' + outcome['fraction'] = 0.0 + outcome['prologuehtml'] = f"""

Sorry but the following {test_is} not in the required form. +Please re-read the question to see what was expected. Ask a tutor if you're still unsure. +

    """ + for code in bad_tests: + outcome['prologuehtml'] += f'
  • {code}
    • \n' + outcome['prologuehtml'] += '' + ok = False + +if not PARAMS['IS_PRECHECK'] and ok: + # Check the Example table cases with the so-called sample answer + #print("Running example tests") + + outcome = run_tests(PARAMS, example_tests) + outcome = build_new_outcome(outcome) + if 'prologuehtml' in outcome: + outcome['prologuehtml'] = "

    ERROR IN QUESTION'S HIDDEN TESTING CODE. PLEASE REPORT

    \n" + outcome['prologuehtml'] + ok = False + elif outcome['fraction'] == 1.0: + outcome['prologuehtml'] = '

    All good!

    ' + else: + outcome['prologuehtml'] = """One or more of your tests is invalid. +
    The erroneous cells are shaded red in the result table.""" + ok = False + +print(json.dumps(outcome))