From 7967d3bc6f7fbc62f72576bcead3b769a3b28253 Mon Sep 17 00:00:00 2001 From: Richard Lobb Date: Sat, 1 Apr 2023 16:16:58 +1300 Subject: [PATCH] Add new python3_scratchpad question type --- .idea/.gitignore | 3 + .idea/inspectionProfiles/Project_Default.xml | 10 + .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 4 + .idea/modules.xml | 8 + .idea/ucquestiontypes.iml | 14 + .idea/vcs.xml | 6 + .../.cache/pylint/__source_1.stats | Bin 0 -> 733 bytes python3_scratchpad/__author_solution.html | 47 ++ .../__author_solution_scrambled.html | 47 ++ python3_scratchpad/__languagetask.py | 77 +++ python3_scratchpad/__plottools.py | 342 ++++++++++++ python3_scratchpad/__plottools.zip | Bin 0 -> 4540 bytes python3_scratchpad/__pystylechecker.py | 491 ++++++++++++++++++ python3_scratchpad/__pytask.py | 255 +++++++++ python3_scratchpad/__resulttable.py | 250 +++++++++ python3_scratchpad/__tester.py | 347 +++++++++++++ python3_scratchpad/__watchdog.py | 27 + python3_scratchpad/pytester.py | 255 +++++++++ 19 files changed, 2189 insertions(+) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/ucquestiontypes.iml create mode 100644 .idea/vcs.xml create mode 100644 python3_scratchpad/.cache/pylint/__source_1.stats create mode 100644 python3_scratchpad/__author_solution.html create mode 100644 python3_scratchpad/__author_solution_scrambled.html create mode 100644 python3_scratchpad/__languagetask.py create mode 100644 python3_scratchpad/__plottools.py create mode 100644 python3_scratchpad/__plottools.zip create mode 100644 python3_scratchpad/__pystylechecker.py create mode 100644 python3_scratchpad/__pytask.py create mode 100644 python3_scratchpad/__resulttable.py create mode 100644 python3_scratchpad/__tester.py create mode 100644 python3_scratchpad/__watchdog.py create mode 100644 python3_scratchpad/pytester.py diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..146ab09 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..62407b5 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..112f56f --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/ucquestiontypes.iml b/.idea/ucquestiontypes.iml new file mode 100644 index 0000000..8adc565 --- /dev/null +++ b/.idea/ucquestiontypes.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/python3_scratchpad/.cache/pylint/__source_1.stats b/python3_scratchpad/.cache/pylint/__source_1.stats new file mode 100644 index 0000000000000000000000000000000000000000..74b65be656e6a427d2a2f0b95d69f88c1bf26983 GIT binary patch literal 733 zcmZWn&1&2*5O%{lYwwmOr7Ye4-Fiqb3+)s1mV_ZSfdml0_|OpLY)`O z6Lm(~B!w6T=I5K4Z$9hS;`Z*ONMGCzlhGD-0krWua@FX8gP-I2TZ+D?=7w3!itZF&?^gbQ^W-a7|AJr816lJXM#LN1>FWvlw0SVaMq``^1-XMMGESQ zrzzDwSOMBuYC7!|bWUd1$zal`3Za^}IatCp9D4)uoLHZ6*K@2G^Dc}+CDJQgc!ivg zqf0f8F7w`V;D&ZAwB0*$s!{E^fLyOh8kNu8#N|&uT5S&q3qGFGLVv&oc(jv>5gR64 zKy6641tn?XQn>zJv06IeQJOR)mZ~3MB1{P`6AAITRD-foS)q@buX+xHabR>i-^>@? z=1*KRMj3F<`1{<;N7!zdsC=x1crL?Ww7^>=Bf;~(d}Ukqq=00$gF@vjJS|@4S5y=s zt}NaBaBlaaIqB-A=+4m2n`d}Ouy|~GD@u|g2r2%96rp>Ki7nb2w6|#Q(B3zj?gNro aAB=0cVb_67C; literal 0 HcmV?d00001 diff --git a/python3_scratchpad/__author_solution.html b/python3_scratchpad/__author_solution.html new file mode 100644 index 0000000..bf702ff --- /dev/null +++ b/python3_scratchpad/__author_solution.html @@ -0,0 +1,47 @@ +
+ Show author's solution +
+
+ 
%s
+
+ + diff --git a/python3_scratchpad/__author_solution_scrambled.html b/python3_scratchpad/__author_solution_scrambled.html new file mode 100644 index 0000000..e481e1c --- /dev/null +++ b/python3_scratchpad/__author_solution_scrambled.html @@ -0,0 +1,47 @@ +
+ Show scrambled author's solution +
+
+ 
%s
+
+ + diff --git a/python3_scratchpad/__languagetask.py b/python3_scratchpad/__languagetask.py new file mode 100644 index 0000000..ba78896 --- /dev/null +++ b/python3_scratchpad/__languagetask.py @@ -0,0 +1,77 @@ +"""The generic LanguageTask, subclasses of which manage compiling and executing + code in a particular language. +""" +from datetime import datetime + +WATCHDOG_FREEBOARD = 1 + +class CompileError(Exception): + def __init__(self, error_message): + Exception.__init__(self, error_message) + + +class RunError(Exception): + def __init__(self, error_message=''): + Exception.__init__(self, error_message) + +class LanguageTask: + def __init__(self, params, code=None): + """Initialise the object, recording the parameters that will control compilation and + running plus the code if supplied. Code may be alternatively be supplied later by + calls to set_code. + self.params is the dictionary of template & global parameters - language specific. + """ + self.params = params + self.code = code + self.executable_built = False + self.compile_error_message = None + self.error_message_offset = 0 + self.stderr = '' + self.stdout = '' + self.start_time = datetime.now() + self.timed_out = False + if 'totaltimeout' not in params: + self.params['totaltimeout'] = 30 # Secs + + def seconds_remaining(self): + """The number of seconds of execution time remaining before the watchdog timer goes off. + The watchdog timer goes off 1 second before runguard kills the job (as determined by the 'timeout' parameter). + """ + t_elapsed = (datetime.now() - self.start_time).total_seconds() + return self.params['totaltimeout'] - t_elapsed - WATCHDOG_FREEBOARD + + def set_code(self, code, error_message_offset=0): + """Set the code to be used for subsequent compiling and running. The optional error_message_offset + is a number to be subtracted from any error messages generated by compile and run_code calls. + Exactly how (or even 'if') it is used is language dependent. + """ + self.code = code + self.error_message_offset = error_message_offset + + def compile(self, make_executable=False): + """Compile the currently set code, either to an object file or + to an executable file depending on the given make_executable parameter. + Adjust any error message by subtracting error_message_offset. + Raise CompileError if the code does not + compile, with the compilation error message within the exception + and also recorded in self.compile_error_message. + No return value. + """ + raise NotImplementedError("compile not implemented by concrete class") + + def discard_executable(self): + """Called if something breaks in the executable and it will need rebuilding + (with different source, presumably) + """ + self.executable_built = False + + def run_code(self, standard_input=None, bash_command=None): + """Run the code in the executable program that a call to compile is assumed + to have created, using the given standard input. + If a bash_command is supplied it used as given. + Otherwise the command to be executed is the compiled executable. + Returns a tuple of the output from the + run and a stderr (or a derivative thereof) string. Those two values + are also recorded in self.stdout and self.stderr respectively. + """ + raise NotImplementedError("run_code not implemented by concrete class") \ No newline at end of file diff --git a/python3_scratchpad/__plottools.py b/python3_scratchpad/__plottools.py new file mode 100644 index 0000000..70590de --- /dev/null +++ b/python3_scratchpad/__plottools.py @@ -0,0 +1,342 @@ +"""Define support functions for testing of matplotlib questions. + The main function is print_plot_info, which displays suitably formatted + data about the current matplotlib plot. + + This module works only if imported *after* a call to matplotlibg.use("Agg") has + been done. +""" +import traceback +import numpy as np +import matplotlib.pyplot as plt +from matplotlib import colors as colors +from scipy import interpolate + +DEFAULT_PARAMS = { + 'x_samples': None, # A list of x-values at which y values should be interpolated. + 'bar_indices': None, # A list of the 0-origin bar indices to report on. None for all bars. + 'show_xlim': False, # True to display the x-axis limits + 'show_ylim': False, # True to display the y-axis limits + 'show_colour': False, # True to report line/marker colour + 'show_xticklabels': None, # True to display x-tick labels (defaults True for bars, False otherwise) + 'show_yticklabels': False, # True to display y-tick labels + 'show_xticks': False, # True to display x-tick numeric values + 'show_yticks': False, # True to display y-tick numeric values + 'show_barx': True, # True to print the x-coordinates of all bars + 'show_linelabels': None, # True to show line labels, default is True if there's a legend else False + 'sort_points': False, # True to sort data by x then y. + 'first_num_points': 5, # Number of points to print at the start of the point list. + 'last_num_points': 5, # Number of points to print at the end of the point list. + 'float_precision': (1, 1), # Num digits to display after decimal point for x and y values resp + 'max_label_length': 60, # Use multiline display if tick label string length exceeds this + 'lines_to_print': None, # If non-None, a list of indices of lines to print (0 is first line). + 'line_info_only': False, # True to suppress all except the line/bar/points info +} + + +class PlotChecker: + """Wrapper for all the internal methods used to print plot info.""" + + def __init__(self, params_dict): + """Initialise with a subset of the options listed above""" + self.params = DEFAULT_PARAMS.copy() + self.params.update(params_dict) + + @staticmethod + def my_interpolate(data, xs): + """Return the spline interpolated list of (x, y) values at abscissa xs, given + a list of (x, y) pairs + """ + + def linear(x, xa, ya, xb, yb): + return ya + (x - xa) / (xb - xa) * (yb - ya) + + if len(data[:, 0]) == 2: + x0, y0 = data[0][0], data[0][1] + x1, y1 = data[-1][0], data[-1][1] + return [(x, linear(x, x0, y0, x1, y1)) for x in xs] + else: # cubic + tck = interpolate.splrep(data[:, 0], data[:, 1], s=0) # Cubic spline interpolator + return zip(xs, interpolate.splev(xs, tck)) # Evaluate at required x values + + @staticmethod + def fmt_float(value, digits_precision=2): + """Return a formatted floating point number to the precision specified, + replacing -0 with 0""" + format_string = f'.{digits_precision}f' + s = format(value, format_string) + if s.startswith('-') and float(s) == 0.0: + s = s[1:] # Strip off the minus sign. + return s + + def fmt_float_axis(self, value, axis): + """Format a given value as a float with the appropriate number of + digits of precision for the specified axis ('x' or 'y'). + Except - string values are returned as is. + """ + if isinstance(value, str): + return value + else: + precision = self.params['float_precision'][0 if axis == 'x' else 1] + return self.fmt_float(value, precision) + + def fmt_float_pair(self, p, precision=None): + """A formatted (x, y) point or other pair of floating-point numbers. + By default, use float_precision_x and float_precision_y for the + first and second numbers resp, else use precision if given. + """ + if precision is None: + x_precision = self.params['float_precision'][0] + y_precision = self.params['float_precision'][1] + else: + x_precision = y_precision = precision + return f"({self.fmt_float(p[0], x_precision)}, {self.fmt_float(p[1], y_precision)})" + + @staticmethod + def normalise_colour(colour): + """Given a matplotlib colour, convert to a standarised format""" + rgb = colors.to_rgb(colour) + return f"RGB({rgb[0]:0.2f}, {rgb[1]:0.2f}, {rgb[2]:0.2f})" + + def print_line(self, line, xsamples): + """Print the info for the given line""" + if self.params['show_colour']: + print("Color:", self.normalise_colour(line.get_color())) + print("Marker:", line.get_marker()) + print("Line style:", line.get_linestyle()) + label = line.get_label() + if label and self.params['show_linelabels']: + print("Label:", label) + data = line.get_xydata() + + if self.params['sort_points']: + data = np.array(sorted([[row[0], row[1]] for row in data])) + print("Plotted data, after sorting ...") + + if xsamples is not None: + print(f"First point: {self.fmt_float_pair(data[0])}") + print(f"Last point: {self.fmt_float_pair(data[-1])}") + print(f"Interpolating line at selected x values:") + interpolated = self.my_interpolate(data, xsamples) + for p in interpolated: + print(' ', self.fmt_float_pair(p)) + else: + print(f"Num points: {len(data)}") + n = min(len(data), self.params['first_num_points']) + if n: + points = '\n '.join(self.fmt_float_pair(p) for p in data[:n]) + if n < len(data): + print(f"First {n} points:\n {points}") + else: + print(f" {points}") + last_n = min(len(data) - n, self.params['last_num_points']) + if last_n: + points = '\n '.join(self.fmt_float_pair(p) for p in data[-last_n:]) + print(f"Last {last_n} points:\n {points}") + + def print_lines(self, subplot, xsamples): + """Print all selected lines in the plot showing y values interplolated at the x sample points, + if given. Otherwise, print just the first first_num_points and last last_num_points. Also + show line colours if the show_colour parameter is True. + """ + lines = subplot.get_lines() + if len(lines) == 0: + print("No plotted lines found") + return + line_indices = self.params['lines_to_print'] + if line_indices is None: + wanted_lines = lines + else: + wanted_lines = [] + for i in line_indices: + if i >= len(lines): + print(f"Can't display info for plot {i} - no such plot!") + return + else: + wanted_lines.append(lines[i]) + + multilines = len(wanted_lines) > 1 + if multilines: + print(f"Displaying info for {len(wanted_lines)} lines") + for i, line in enumerate(wanted_lines, 1): + if multilines: + print(f"Line {i}:") + self.print_line(line, xsamples) + if multilines: + print() + + @staticmethod + def in_range(labels, limits): + """Return the list of axis labels, filtered to include only those within + the given limits (min, max). If any of the axis labels are non-numeric + the list is returned unchanged. + """ + try: + clipped_labels = [] + for s in labels: + if isinstance(s, str): + s = s.replace('−', '-') + if limits[0] <= float(s) <= limits[1]: + clipped_labels.append(s) + return clipped_labels + except ValueError: + return labels + + def print_bars(self, subplot): + """Print a list of all bars if the bar_indices param is None or a list of the + bars with the given indices, otherwise. + """ + print("Bars:") + bars = subplot.patches + if bars and self.params['show_colour']: + print(f"First bar colour: {self.normalise_colour(bars[0].get_facecolor())}") + bar_indices = self.params['bar_indices'] + if bar_indices is None: + bar_indices = range(0, len(subplot.patches)) + for i in bar_indices: + try: + bar = subplot.patches[i] + y = bar.get_height() + if self.params['show_barx']: + x = bar.get_xy()[0] + bar.get_width() / 2 + bar_spec = f"Bar{i}: x = {self.fmt_float_axis(x, 'x')}, height = {self.fmt_float_axis(y, 'y')}" + else: + bar_spec = f"Bar{i}: height = {self.fmt_float_axis(y, 'y')}" + print(bar_spec) + except IndexError: + print(f"Bar{i} not found. Number of bars = {len(subplot.patches)}") + break + + def tick_label_text(self, labels): + """Return a string suitable for displaying tick labels (multiline or + single line depending on length. + """ + label_text = ', '.join(labels) + if len(label_text) > self.params['max_label_length']: + label_text = '\n'.join(labels) + return label_text + + def print_ticks_for_axis(self, axis, subplot, axis_limit): + """Print tick and ticklabel info for the given axis ('x' or 'y') of + the given subplot. axis_limit is either xlim or ylim. + """ + axis = axis.lower() # Just to be safe + if axis == 'x': + ticks = subplot.get_xticks() + tick_labels_obj = subplot.get_xticklabels() + else: + assert axis == 'y' + ticks = subplot.get_yticks() + tick_labels_obj = subplot.get_yticklabels() + + tick_labels = [label.get_text() for label in tick_labels_obj] + + if all(float(int(tick)) == tick for tick in ticks): + # If all ticks are integers, don't format as floats + formatted_ticks = [str(int(tick)) for tick in ticks] + else: + formatted_ticks = [self.fmt_float_axis(pos, axis) for pos in ticks] + + if self.params[f'show_{axis}ticks']: + print(f'{axis.upper()}-axis ticks at ', ', '.join(formatted_ticks)) + + if self.params[f'show_{axis}ticklabels']: + # A problem here is that in a call to bar(axis_labels, bar_heights) the call to get_xticklabels doesn't + # return the actual labels, but rather their tick locations. I can't find a workaround for this. + if all(label.strip() == '' for label in tick_labels): + tick_labels = formatted_ticks + tick_labels = self.in_range(tick_labels, axis_limit) + print(f'{axis.upper()}-axis tick labels:') + print(self.tick_label_text(tick_labels)) + + + def print_ticks(self, subplot, xlim, ylim): + """Print tick and ticklabel info for the given subplot.""" + self.print_ticks_for_axis('x', subplot, xlim) + self.print_ticks_for_axis('y', subplot, ylim) + + def print_axis_info(self, subplot): + """Print the axis info for the given subplot""" + + print("X-axis label: '{}'".format(subplot.get_xlabel())) + print("Y-axis label: '{}'".format(subplot.get_ylabel())) + xgridlines = subplot.get_xgridlines() + ygridlines = subplot.get_ygridlines() + gridx_on = len(xgridlines) > 0 and bool(xgridlines[0].get_visible()) + gridy_on = len(ygridlines) > 0 and bool(ygridlines[0].get_visible()) + print(f"(x, y) grid lines enabled: ({gridx_on}, {gridy_on})") + xlim = subplot.get_xlim() + ylim = subplot.get_ylim() + if self.params['show_xlim']: + print(f"X-axis limits: {self.fmt_float_pair(xlim, precision=self.params['float_precision'][0])}") + if self.params['show_ylim']: + print(f"Y-axis limits: {self.fmt_float_pair(ylim, precision=self.params['float_precision'][1])}") + self.print_ticks(subplot, xlim, ylim) + + if subplot.get_legend() is not None: + print(f"Legend: True") + print() + + def print_subplot_info(self, data_type, subplot, title): + """Print the info for a single given subplot. + If the data_type is 'lines' an optional parameter x_samples + can be used to specify x values at which the line should be sampled. + If the data_type is lines, the x-tick labels are shown unless + the show_xticklabels parameters is explicitly set to False. + """ + if not self.params['line_info_only']: + if self.params['show_xticklabels'] is None and data_type == 'bars': + self.params['show_xticklabels'] = True + has_legend = subplot.get_legend() is not None + if has_legend and self.params['show_linelabels'] is None: + self.params['show_linelabels'] = True + print("Plot title: '{}'".format(title)) + self.print_axis_info(subplot) + + if data_type == 'points': + self.print_lines(subplot, None) + elif data_type == 'lines': + self.print_lines(subplot, self.params['x_samples']) + elif data_type == 'bars': + self.print_bars(subplot) + + def print_info(self, data_type): + """Print all the information for the current plot. data_type + must be 'points', 'lines' or 'bars'. + """ + try: + axes = plt.gcf().get_axes() + texts = plt.gcf().texts + if not self.params['line_info_only']: + if len(axes) > 1: + print(f"Figure has {len(axes)} subplots") + if len(texts) != 0: + print(f"Suptitle: {texts[0].get_text()}\n") + for i, current_axes in enumerate(axes, 1): + if len(axes) > 1 and not self.params['line_info_only']: + print(f"Subplot {i}\n---------") + subplot = current_axes.axes + title = current_axes.title.get_text() + self.print_subplot_info(data_type, subplot, title) + if len(axes) > 1 and not self.params['line_info_only']: + print(40 * "=") + print() + + except Exception as e: + print("Failed to get plot info:", repr(e)) + traceback.print_exception(e) + + +def print_plot_info(data_type, **kwparams): + """Output key attributes of current plot, as defined by plt.gca(). + data_type must be one of 'points', 'lines' or 'bars', to print the + appropriate type of data. + For values of possible keyword parameters see DEFAULT_PARAMS declaration + """ + + unknown_params = set(kwparams) - set(DEFAULT_PARAMS) + if unknown_params: + print(f"Unknown parameter(s) passed to print_plot_info: {', '.join(unknown_params)}") + return + + checker = PlotChecker(kwparams) + checker.print_info(data_type) diff --git a/python3_scratchpad/__plottools.zip b/python3_scratchpad/__plottools.zip new file mode 100644 index 0000000000000000000000000000000000000000..c2349801998e15adc169d9b512c59f77e061315a GIT binary patch literal 4540 zcmb7|_ct7ho5shC7G-oHg6N`mqW6SR6J-b)qh&CI(Q6RBbG2M8I*D$S=tM6Qy+@Bu z5WVlcd-i*Ff8FOfZ#&O<|ACJl1cXZsApFOYG15kW|B<2$000I&1HfR;NGCTpCnuz< zfU~!u0V#msD{$QEKjrR448XzN1_1#7-C7wI;@0cW7Ga%DWT_HNHnwBu@PtQAbJoH9 zE-C`3kCE_Ekpq3BRBDZ-YUxiG@)uM$9CD9evpyfhG;=o*iq!adIm&vDpIqI#)(sJH zq+1!Lxbu4Sv7#_Fi0+CqwbFD6-t)Ylk-^B)7UT(%?SsOaL!gGOas=}swcp<~cdrx0 z^S)y8%-27<8|@Dkerq2Zm#N04^Q4)Be$^N`{sDghY#N~`ri|)e_iKA2)rg@JR_3L7 zbsFo-*1gt2!?{11TseLZU2C%q0*`ED5*hU<>u5%PRT!R$vZ(Q#nAH}B+I@b?K!b(t zUz|XH4&duMCi8UH^BIU_jBJVavdxB4>pI{ zdQ_?oOYdT13-=z9>;^oltdbw+pmv+4rp!AKr*{XKM%*gie()c}fGh$yPQu0>pv*L^ zqw314I*Go zPB$;oBQn*-1P_a=i>)OkD3*}%*4gtEp%aS5QN7Z}J#DQZ)__KKphD0zS(PL`>Jv*K z!{l3G?!w-dE=Z_!p(ZMbIpWHczpB=xVI$qTUVH6KhuQ|+`PqiKS}XA5OH-L`-X>MH z+`IhBHS1?e0}io~97ys6F{QOI657AZpcmBg9f1ndzQIu2c+4DM=mE0ZGlAsY(iiHy zA@Q+~$dhF-rDpgd~|yDN{S&=M5ik1xPPgk#_vg+t3gaBGu1A9}~6l*4PyJ1OU>s%w6m13JO&E;Uifw1i%&M6;k$xgR|FW6_#jMw(RB--4*yE z2a+Jjyke!@=E1Md1>U*Q%nR~G+u*DdkeQ15M~|DhP_`EK()3BQE9yJjhn>;AuN-q4 zU+07{*7AGZ#pk$HNCMaFsYp2-M{H0cjuit|nH(RUkq_e7@g!tquPoyUxs zS*+iba4EYc1l}xc3@1`qBBDwK1?DmYQN}hr4vYp)-HwT}mwfWSX@i?tNy>$RFX${6 zxHCd1p}_|EJw2>NAszFGes#el$c!e1zpJl%t8(E3fSdP77%UxLCZ&u{KZq`XHQ zTdJ34pxZlv#V>j#E5>1oBsp#_Zed-NW+Dw`lGZ?0w7GJO+8b`I>^hFo9f~BUcHv0@ z{o64H)u#~%v?<}G{TTmRckeM?aWrw#u9n`{6k0lA#a#p?*9tV9pb2lg?N!Pghayd! z_-X52Jb3n5Z!BY})5{bf?jv^Xyw2%7X9Km+e7xKzFJ8v;XXANLT9s*-)9#aUx_K(V zpi5Rju`6&%#;#q|%r{7Hg8*{bKJ(zROq64ta-!@A7xoz+Q@`Y4yMQt<(|qQYKTdg% z@G*GGZvuN|sV9v{M!L9^@{fjTG79cjF24sYmhUdTKbYE$p=dPw=5_OaxzzJyyHhkN ziZU^j|ASeSFu85;5B#XDElanc`j1m_jt{K&>*- znjck6R&2GBWlK(q*1z|uO7vGTxe~MwH-ZY$OaNd=Tbow)p`OvL*Tx6G3WZ*y!afz+ z;lO$L5k)jx_M>lk{xc14p(5w2QC2Jhr&>R%ahp-^u5v*~>L+R32WNf*d+LMgz=3yl zr;TxIUZWxEN8;`a5xLe8GQ&{j&T(ftE(k}jq_2ueH~ zKG6PMFp|uV?&x(sVoc8clG6nuot8Zn;8QDBlc#BIJ|tj|_)fF9G`%6$NPORFT8@CN z(nEyIFg0TE;}}N5Fzt$>#s-z-PZ58y|J|(hB>4Nv8^sY}uwsmdWPn<3yy~}e^HjEG z0rmAh?}%zf#ECh%Vvuy?sypnLK0#y=aDY+oOKsh5qk~i9gwVF4)R-^g!X-6-iFdLo zw$E_9;}`mQXA-#Rh_OiD`24BJV7P&x`qM&0_A}JkcI<{u7b@b| zZ>JTMOTAdlOkJJvYkv83c$fXt$IWp&^z{=BKL^`Fh#cY7!R(PwMT;M%s3YzVu~R(d zF{_hiM!(R@UoBJHjk#5n&vXr_!?fhgNYYScAq@F#EGpiO1jwj&u9ehU!pq48JxX=W zf1pS?$VSw67N@Yw5AZGR5hJj|d6Pw*w;=qm-hqwzPL2vAd$cgc?P!bOkyxybw`gXQ zZ>_F%U&@CKT8@D+YHyAc_~otRx7(%KAcxyi%s_p%oN+sPl^RZ|QQrFD->CW;7kh;} z_Pd~0=S@&Qo{D`&`kbzT71vbC#nQ*8@Q|}a$-%#p4MsMo90T(n$#1^!WsD20yn7|l zlZd66Lgd&hjMT7zQ%a37Z3DzH*vEaw5P~hVtzk!&E3=Ys>KE?#3V(# z!${OXN|00k+bpZl8=te$+`&Z&R#1fnzy+U%fwd7UqfVl|nIT)Rc}#D&U=Lv*3Hnr6 z3d%URiF>#tySBHJ7*2Yoc(l4{`lZaMqN17h$t%4`PCSP-VUzAA1Y2A_iD~S|5rf#o z&S5FK6!~cpjUH*+jRMOxejTAOnEloyj(Y~Gq1Rd^RdneOMK!sX&Iq4V1Aoo8{5pB|onbJW0$^<715 zx}}r4{K>VReNC!qJR~jMB2d3e9Rcog>B2wjGsxnYrYJwJ3{TJwl}^lS?}bxy0k1nD!k@+Jf~mwRi_p( zk)+shG12rn{nZ5(tEqLmlkMACj9uDjWe+ZkOdyq*|2m#a)2}Lg#C0hH6 zTgq4^)Y5Oi5*gBq%%{RRG=m3<8CfRwY!`~8JFHL9s|Gb#d|&7Q`CKRc+K|&3JF41n zntYz(bx@orW8)MqK7IkD=jq$Cy0nda{1?6`YIx9x(>96SNi!Ba^{NA^O)Dm25;2)&7D%-%^1|A0gr+sqJ zIov8;QZ>)aDJJJTHkdk*@e^X8Mx0kPYZUXk87f+9A%FNL z%*GX&n~fcGw9_X$-gOb)F*-YA~QMIj|Jo z!4GQdIOBGDT$DLO?c>k&-hWGV<3faJVvSFEC?fl_8)|GVaq7Ap=Ouf<2mBa;=ZF9= zB4H(@9~D>fz_U-jr*|?hIt{a0qb|=j*jm|qbuSeB(`5K#U6a%=xD{X|<;QhUSy_S2 z$hM@F0vi8JQF-?3jYV=dE8JrLER@Q6^C~f~o!gsHbHXRfS<`NuQFx<)6vR1@`3G&2 z&FRNQ&wYr?FM8z`ls`EtPHVI5E#dwB-1+4=Lzz#Erh5we=eEci~Er^Odvh13>N z!R3H_y-c)dhFoM3ZxwmZXW5CHsApHe!=I9A`WB;N`?G-rG<)@>$utJ@W!R}Cg30mI zXO8=rIX`TEayI^8*U16iYGbvRA!~P}%xu4iTC!P%ymu{;0Epzz2>f$kj2-F3cI#Svk|T`0V&q=N)I~ z0TUwuRMK278tj{IPI6|&OClBA4`sI4YNj&=>aw)3IJl-R4yZGf2NY3OS;GJh#}mc3 ztHS!9Q18tH9rjwXP)kFq`vgf;F1*-X+&!t1)law?p&N&wh|5D7HEWP=`z+H#*=^QaQAXn z8)Ayx6A!9R>DAruT_ek%(E9wHtWWc%8&odb^GI?wzMYnd)Ze1eGuCa3$!s;_ILPpo z8xe>(M>*{-_J!_k-8Z2JpwAdjmkgYC@6o;BC|i)S-zy)xFSe{gu~bKHyLSqRZQZoB zzdtQ&WtK~yf$k?TMs1Iy-w}m9=F`}$v`%%+#!n$p2=0)9)_vXWCo8!VysV8%X2D6o z%poJ5EA`0vo^NLN#NWUi{CttT+tte@U>*=6Ja=_-Idzu8CXy1lvz7ApwufA%HYUus zLq$jo9tIBZ*vi9>UmYZOTEk2@LpZ|cvnP2Ka~$Zev3e?J3Yq}y!*m7rlfo95>&nOn z7XSi#hi!w&&^i&7yq&%qQOp5E`na!d*k10lu9xFDxXyqXP(Sy$?mt+Nj^Oe>B?Gqy z(R+|s_6x0`^<@k(^5exL&mZs!X6rvBidH#wu^Wxkk7M`S^&kXewc=-=)CM^^M8c=-Q*;rwH>f2JnIzt?{Ott5hN literal 0 HcmV?d00001 diff --git a/python3_scratchpad/__pystylechecker.py b/python3_scratchpad/__pystylechecker.py new file mode 100644 index 0000000..acfe76f --- /dev/null +++ b/python3_scratchpad/__pystylechecker.py @@ -0,0 +1,491 @@ +"""All the style checking code for Python3""" + +from io import BytesIO +import os +import sys +import subprocess +import ast +import tokenize +import token +import re +import shutil +from collections import defaultdict + +class StyleChecker: + def __init__(self, prelude, student_answer, params): + self.prelude = prelude + self.student_answer = student_answer + self.params = params + self.function_call_map = None + self._tree = None + + @property + def tree(self): + if self._tree is None: + self._tree = ast.parse(self.student_answer) + return self._tree + + def style_errors(self): + """Return a list of errors from local style checks plus pylint and/or mypy + """ + errors = [] + source = open('__source.py', 'w', encoding="utf-8") + code_to_check = self.prelude + self.student_answer + prelude_len = len(self.prelude.splitlines()) + source.write(code_to_check) + source.close() + env = os.environ.copy() + env['HOME'] = os.getcwd() + pylint_opts = self.params.get('pylintoptions',[]) + precheckers = self.params.get('precheckers', ['pylint']) + result = '' + + if 'pylint' in precheckers: + try: # Run pylint + cmd = f'{sys.executable} -m pylint ' + ' '.join(pylint_opts) + ' __source.py' + result = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env, + shell=True) + except Exception as e: + result = e.output + + else: + # (mct63) Abort if there are any comments containing 'pylint:'. + try: + tokenizer = tokenize.tokenize(BytesIO(self.student_answer.encode('utf-8')).readline) + for token_type, token_text, *_ in tokenizer: + if token_type == tokenize.COMMENT and 'pylint:' in token_text: + errors.append("Comments can not include 'pylint:'") + break + + except Exception: + errors.append("Something went wrong while parsing comments. Report this.") + + if "Using config file" in result: + result = '\n'.join(result.splitlines()[1:]).split() + + if result == '' and 'mypy' in precheckers: + code_to_check = 'from typing import List as list, Dict as dict, Tuple as tuple, Set as set, Any\n' + code_to_check + with open('__source2.py', 'w', encoding='utf-8') as outfile: + outfile.write(code_to_check) + cmd = f'{sys.executable} -m mypy --no-error-summary --no-strict-optional __source2.py' + try: # Run mypy + subprocess.check_output(cmd, # Raises an exception if there are errors + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env, + shell=True) + except Exception as e: + result = e.output + line_num_fix = lambda match: "Line " + str(int(match[1]) - 1 - prelude_len) + match[2] + result = re.sub(r'__source2.py:(\d+)(.*)', line_num_fix, result) + + if result == '' and self.params.get('requiretypehints', False): + bad_funcs = self.check_type_hints() + for fun in bad_funcs: + result += f"Function '{fun}' does not have correct type hints\n" + + if result: + errors = result.strip().splitlines() + + return errors + + def prettied(self, construct): + """Expand, if possible, the name of the given Python construct to a more + user friendly version, e.g. 'listcomprehension' -> 'list comprehension' + """ + expanded = { + 'listcomprehension': 'list comprehension', + 'while': 'while loop', + 'for': 'for loop', + 'try': 'try ... except statement', + 'dictcomprehension': 'dictionary comprehension', + 'slice': 'slice' + } + if construct in expanded: + return expanded[construct] + else: + return f"{construct} statement" + + def local_errors(self): + """Perform various local checks as specified by the current set of + template parameters. + """ + errors = [] + + for banned in self.params.get('proscribedsubstrings', []): + if banned in self.student_answer: + errors.append(f"The string '{banned}' is not permitted anywhere in your code.") + + for required in self.params.get('requiredsubstrings', []): + if isinstance(required, str) and required not in self.student_answer: + errors.append(f'The string "{required}" must occur somewhere in your code.') + elif isinstance(required, dict): + if 'pattern' in required and not re.findall(required['pattern'], self.student_answer): + errors.append(required['errormessage']) + elif 'string' in required and required['string'] not in self.student_answer: + errors.append(required['errormessage']) + + if self.params.get('banglobalcode', True): + errors += self.find_global_code() + + if not self.params.get('allownestedfunctions', True): + # Except for legacy questions or where explicitly allowed, nested functions are banned + nested_funcs = self.find_nested_functions() + for func in nested_funcs: + errors.append("Function '{}' is defined inside another function".format(func)) + + max_length = self.params['maxfunctionlength'] + bad_funcs = self.find_too_long_funcs(max_length) + for func, count in bad_funcs: + errors.append("Function '{}' is too long\n({} statements, max is {})" + "".format(func, count, max_length)) + + bad_used = self.find_illegal_functions() + for name in bad_used: + errors.append("You called the banned function '{}'.".format(name)) + + missing_funcs = self.find_missing_required_function_calls() + for name in missing_funcs: + errors.append("You forgot to use the required function '{}'.".format(name)) + + missing_funcs = self.find_missing_required_function_definitions() + for name in missing_funcs: + errors.append("You forgot to define the required function '{}'.".format(name)) + + missing_constructs = self.find_missing_required_constructs() + for reqd in missing_constructs: + expanded = self.prettied(reqd) + errors.append(f"Your program must include at least one {expanded}.") + + bad_constructs = self.find_illegal_constructs() + for notallowed in bad_constructs: + expanded = self.prettied(notallowed) + errors.append(f"Your program must not include any {expanded}s.") + + num_constants = len([line for line in self.student_answer.split('\n') if re.match(' *[A-Z_][A-Z_0-9]* *=', line)]) + if num_constants > self.params['maxnumconstants']: + errors.append("You may not use more than " + str(self.params['maxnumconstants']) + " constants.") + + # (mct63) Check if anything restricted is being imported. + if 'restrictedmodules' in self.params: + restricted = self.params['restrictedmodules'] + for import_name, names in self.find_all_imports().items(): + if import_name in restricted: + if restricted[import_name].get('onlyallow', None) == []: + errors.append("Your program should not import anything from '{}'.".format(import_name)) + else: + for name in names: + if (('onlyallow' in restricted[import_name] and name not in restricted[import_name]['onlyallow']) or + name in restricted[import_name].get('disallow', [])): + errors.append("Your program should not import '{}' from '{}'.".format(name, import_name)) + + return errors + + def find_all_imports(self): + """Returns a dictionary mapping in which the keys are all modules + being imported and the values are a list of what things within + the module are being modules. An empty list indicates the entire + module is imported.""" + found_imports = {} + class ImportFinder(ast.NodeVisitor): + def visit_Import(self, node): + for alias in node.names: + if alias.name not in found_imports: + found_imports[alias.name] = [] + self.generic_visit(node) + def visit_ImportFrom(self, node): + if node.module not in found_imports: + found_imports[node.module] = [] + for alias in node.names: + found_imports[node.module].append(alias.name) + self.generic_visit(node) + + visitor = ImportFinder() + visitor.visit(self.tree) + return found_imports + + def find_all_function_calls(self): + """Return a dictionary mapping in which the keys are all functions + called by the source code and values are a list of + (line_number, nesting_depth) tuples.""" + class FuncFinder(ast.NodeVisitor): + + def __init__(self, *args, **kwargs): + self.depth = 0 + self.found_funcs = defaultdict(list) + super().__init__(*args, **kwargs) + + def visit_FunctionDef(self, node): + """ Every time we enter a function, we get 'deeper' into the code. + We want to note how deep a function is when we find its call.""" + self.depth += 1 + self.generic_visit(node) + self.depth -= 1 + + def visit_Call(self, node): + """A function has been called, so check its name + against the given one.""" + try: + if 'id' in dir(node.func): + name = node.func.id + else: + name = node.func.attr + # Line numbers are 1-indexed, so decrement by 1 + self.found_funcs[name].append((node.lineno - 1, self.depth)) + except AttributeError: + pass # either not calling a function (??) or it's not named. + self.generic_visit(node) + + if self.function_call_map is None: + visitor = FuncFinder() + visitor.visit(self.tree) + self.function_call_map = visitor.found_funcs + return self.function_call_map + + + def find_defined_functions(self): + """Find all the functions defined.""" + defined = set() + class FuncFinder(ast.NodeVisitor): + + def __init__(self): + self.prefix = '' + + def visit_ClassDef(self, node): + old_prefix = self.prefix + self.prefix += node.name + '.' + self.generic_visit(node) + self.prefix = old_prefix + + def visit_FunctionDef(self, node): + defined.add(self.prefix + node.name) + old_prefix = self.prefix + self.prefix += node.name + '.' + self.generic_visit(node) + self.prefix = old_prefix + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + visitor = FuncFinder() + visitor.visit(self.tree) + return defined + + + def constructs_used(self): + """Return a set of all constructs encountered in the parse tree""" + constructs_seen = set() + class ConstructFinder(ast.NodeVisitor): + def visit_Assert(self, node): + constructs_seen.add('assert') + self.generic_visit(node) + def visit_Raise(self, node): + constructs_seen.add('raise') + self.generic_visit(node) + def visit_Lambda(self, node): + constructs_seen.add('lambda') + self.generic_visit(node) + def visit_Import(self, node): + constructs_seen.add('import') + self.generic_visit(node) + def visit_ImportFrom(self, node): + constructs_seen.add('import') + self.generic_visit(node) + def visit_For(self, node): + constructs_seen.add('for') + self.generic_visit(node) + def visit_While(self, node): + constructs_seen.add('while') + self.generic_visit(node) + def visit_Comprehension(self, node): + constructs_seen.add('comprehension') + self.generic_visit(node) + def visit_ListComp(self, node): + constructs_seen.add('listcomprehension') + self.generic_visit(node) + def visit_SetComp(self, node): + constructs_seen.add('setcomprehension') + self.generic_visit(node) + def visit_DictComp(self, node): + constructs_seen.add('dictcomprehension') + self.generic_visit(node) + def visit_Slice(self, node): + constructs_seen.add('slice') + def visit_If(self, node): + constructs_seen.add('if') + self.generic_visit(node) + def visit_Break(self, node): + constructs_seen.add('break') + self.generic_visit(node) + def visit_Continue(self, node): + constructs_seen.add('continue') + self.generic_visit(node) + def visit_Try(self, node): + constructs_seen.add('try') + self.generic_visit(node) + def visit_TryExcept(self, node): + constructs_seen.add('try') + constructs_seen.add('except') + self.generic_visit(node) + def visit_TryFinally(self, node): + constructs_seen.add('try') + constructs_seen.add('finally') + self.generic_visit(node) + def visit_ExceptHandler(self, node): + constructs_seen.add('except') + self.generic_visit(node) + def visit_With(self, node): + constructs_seen.add('with') + self.generic_visit(node) + def visit_Yield(self, node): + constructs_seen.add('yield') + self.generic_visit(node) + def visit_YieldFrom(self, node): + constructs_seen.add('yield') + self.generic_visit(node) + def visit_Return(self, node): + constructs_seen.add('return') + self.generic_visit(node) + + visitor = ConstructFinder() + visitor.visit(self.tree) + return constructs_seen + + def check_type_hints(self): + """Return a list of the names of functions that don't have full type hinting.""" + unhinted = [] + class MyVisitor(ast.NodeVisitor): + def visit_FunctionDef(self, node): + if node.returns is None or any([arg.annotation is None for arg in node.args.args]): + unhinted.append(node.name) + + visitor = MyVisitor() + tree = self.tree + visitor.visit(self.tree) + return unhinted + + def find_function_calls(self, name): + """Look for occurances of a specific function call""" + return self.find_all_function_calls().get(name, []) + + + def find_illegal_functions(self): + """Find a set of all the functions that the student uses + that they are not allowed to use. """ + func_calls = self.find_all_function_calls() + return func_calls.keys() & set(self.params['proscribedfunctions']) + + + def find_missing_required_function_calls(self): + """Find a set of the required functions that the student fails to use""" + func_calls = self.find_all_function_calls() + return set(self.params['requiredfunctioncalls']) - func_calls.keys() + + + def find_missing_required_function_definitions(self): + """Find a set of required functions that the student fails to define""" + func_defs = self.find_defined_functions() + return set(self.params['requiredfunctiondefinitions']) - func_defs + + + def find_illegal_constructs(self): + """Find all the constructs that were used but not allowed""" + constructs = self.constructs_used() + return constructs & set(self.params['proscribedconstructs']) + + + def find_missing_required_constructs(self): + """Find which of the required constructs were not used""" + constructs = self.constructs_used() + return set(self.params['requiredconstructs']) - constructs + + + def find_too_long_funcs(self, max_length): + """Return a list of the functions that exceed the given max_length + Each list element is a tuple of the function name and the number of statements + in its body.""" + + bad_funcs = [] + + class MyVisitor(ast.NodeVisitor): + + def visit_FunctionDef(self, node): + + def count_statements(node): + """Number of statements in the given node and its children""" + count = 1 + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Str): + count = 0 + else: + for attr in ['body', 'orelse', 'finalbody']: + if hasattr(node, attr): + children = node.__dict__[attr] + count += sum(count_statements(child) for child in children) + return count + + num_statements = count_statements(node) - 1 # Disregard def itself + if num_statements > max_length: + bad_funcs.append((node.name, num_statements)) + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + visitor = MyVisitor() + visitor.visit(self.tree) + return bad_funcs + + + def find_global_code(self): + """Return a list of error messages relating to the existence of + any global assignment, for, while and if nodes. Ignores + global assignment statements with an ALL_CAPS target.""" + + global_errors = [] + class MyVisitor(ast.NodeVisitor): + def visit_Assign(self, node): + if node.col_offset == 0: + if len(node.targets) > 1 or isinstance(node.targets[0], ast.Tuple): + global_errors.append(f"Multiple targets in global assignment statement at line {node.lineno}") + elif not node.targets[0].id.isupper(): + global_errors.append(f"Global assignment statement at line {node.lineno}") + + def visit_For(self, node): + if node.col_offset == 0: + global_errors.append(f"Global for loop at line {node.lineno}") + + def visit_While(self, node): + if node.col_offset == 0: + global_errors.append(f"Global while loop at line {node.lineno}") + + def visit_If(self, node): + if node.col_offset == 0: + global_errors.append(f"Global if statement at line {node.lineno}") + + visitor = MyVisitor() + visitor.visit(self.tree) + return global_errors + + + def find_nested_functions(self): + """Return a list of functions that are declared with non-global scope""" + bad_funcs = [] + + class MyVisitor(ast.NodeVisitor): + is_visiting_func = False + + def visit_FunctionDef(self, node): + if self.is_visiting_func: + bad_funcs.append(node.name) + self.is_visiting_func = True + self.generic_visit(node) # Visit all children recursively + self.is_visiting_func = False + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + visitor = MyVisitor() + visitor.visit(self.tree) + return bad_funcs diff --git a/python3_scratchpad/__pytask.py b/python3_scratchpad/__pytask.py new file mode 100644 index 0000000..3cb56bf --- /dev/null +++ b/python3_scratchpad/__pytask.py @@ -0,0 +1,255 @@ +""" Code for compiling (N/A) and running a Python3 task. +""" +import __languagetask as languagetask +import io +import sys +import traceback +import types +from math import floor +import os +import re +from __watchdog import Watchdog + +SOURCE_FILENAME = 'student_answer.py' +DEFAULT_TIMEOUT = 3 # secs +DEFAULT_MAXOUTPUT = 100000 # 100 kB + +class OutOfInput(Exception): + pass + +class ExcessiveOutput(Exception): + pass + +# (mct) New exception for handling situations where submitted code does something it should not. +class InvalidAction(Exception): + def __init__(self, error_message=''): + Exception.__init__(self, error_message) + +def name_matches_res(name, re_strings): + return bool(re.match(f"^{'$|^'.join(re_strings)}$", name)) + +class CodeTrap(object): + """ A safe little container to hold the student's code and grab + its output, while also reformatting exceptions to be nicer. + """ + + def __init__(self, student_code, params, seconds_remaining=None): + self.params = params + if 'timeout' not in params: + self.params['timeout'] = DEFAULT_TIMEOUT + if 'maxoutputbytes' not in params: + self.params['maxoutputbytes'] = DEFAULT_MAXOUTPUT + if 'echostandardinput' not in params: + self.params['echostandardinput'] = True + self.run_code = student_code + self.scoped_globals = self._get_globals() + + if seconds_remaining is None: + self.seconds_remaining = self.params['timeout'] + else: + self.seconds_remaining = min(seconds_remaining, self.params['timeout']) + CodeTrap.MAX_OUTPUT_CHARS = 30000 + CodeTrap.output_chars = 0 # Count of printed chars (more or less) + + def _get_globals(self): + """ Here we define any globals that must be available """ + # change the default options for 'open'. + global np # May not actually be defined but we'll check soon + def new_open(file, mode='r', buffering=-1, + encoding='utf-8', errors=None, + newline=None, closefd=True, opener=None): + + # (mct63) Only open allowed files. + if ('restrictedfiles' in self.params + and ('onlyallow' not in self.params['restrictedfiles'] or name_matches_res(file, self.params['restrictedfiles']['onlyallow'])) + and not name_matches_res(file, self.params['restrictedfiles'].get('disallow', []))): + return open(file, mode, buffering, encoding, errors, newline, closefd, opener) + else: + raise InvalidAction(f"You are not allowed to open '{file}'.") + + # (mct63) Function that creates stub invalid function. + def create_invalid_func(name): + def invalid_func(*args, **kwargs): + raise InvalidAction(f"You are not allowed to use '{name}'!") + return invalid_func + + # (mct63) Checks what is being imported and makes sure it is allowed. If it is not, the thing that + # is not allowed is replaced by an 'invalid function'. If it is an attribute then it is + # not included since I could not think of a better thing to do. Could let it raise + # 'AttributeNotFound' exception and then check if this was caused from removing the + # attribute from the module but given how unlikely this is its not worth it at this time. + def new_import(name, *args, **kwargs): + module = __import__(name, *args, **kwargs) + restricted_module = module + if 'restrictedmodules' in self.params and name in self.params['restrictedmodules']: + NewModuleType = type('module', (types.ModuleType,), {}) + restricted_module = NewModuleType(name) + for var in dir(module): + if (('onlyallow' not in self.params['restrictedmodules'][name] or + name_matches_res(var, self.params['restrictedmodules'][name]['onlyallow'])) and + not name_matches_res(var, self.params['restrictedmodules'][name].get('disallow', []))): + setattr(restricted_module, var, getattr(module, var)) + elif callable(getattr(module, var)): + setattr(restricted_module, var, create_invalid_func(f'{name}.{var}')) + else: + try: + setattr(NewModuleType, var, property(create_invalid_func(f'{name}.{var}'))) + except TypeError: + # Some attributes can not be set to a property so we ignore them. + continue + + return restricted_module + + # (mct63) Insure print always prints to the redirected stdout and not actual stdout. + # (rjl83) Also keep track of print quantity and raise ExcessiveOutput if too much is generated. + def new_print(*values, sep=' ', end='\n', file=None, flush=False): + for value in values: + try: + CodeTrap.output_chars += len(str(value)) + except: + pass + if CodeTrap.output_chars > self.params['maxoutputbytes']: + raise ExcessiveOutput() + return print(*values, sep=sep, end=end, file=sys.stdout) + + # force 'input' to echo to stdin to stdout + if self.params['echostandardinput']: + def new_input(prompt=''): + """ Replace the standard input prompt with a cleverer one. """ + try: + s = input(prompt) + except EOFError: + raise OutOfInput() + print(s) + return s + else: + new_input = input + + # (mct63) Create a new builtins dictionary, redfining any functions that are not allowed. + new_builtins = {key:value for key, value in __builtins__.items()} + new_builtins['open'] = new_open + new_builtins['input'] = new_input + new_builtins['print'] = new_print + new_builtins['__import__'] = new_import + + if 'proscribedbuiltins' in self.params: + for func in self.params['proscribedbuiltins']: + new_builtins[func] = create_invalid_func(func) + + # This would be nice but it can mess with testing code. + # for func in self.params['proscribedfunctions']: + # new_builtins[func] = create_invalid_func(func) + + global_dict = { + '__builtins__': new_builtins, + '__name__': '__main__' + } + if 'usesnumpy' in self.params and self.params['usesnumpy']: + import numpy as np + global_dict['np'] = np + + return global_dict + + def __enter__(self): + if 'MPLCONFIGDIR' not in os.environ or os.environ['MPLCONFIGDIR'].startswith('/home'): + import tempfile + os.environ['MPLCONFIGDIR'] = tempfile.mkdtemp() + self.old_stdout = sys.stdout + self.old_stderr = sys.stderr + self.old_path = os.environ["PATH"] + sys.stdout = io.StringIO() + sys.stderr = io.StringIO() + os.environ["PATH"] = '' # (mct63) Get rid of PATH to make it harder to execute commands. + return self + + def __exit__(self, *args): + sys.stdout = self.old_stdout + sys.stderr = self.old_stderr + os.environ["PATH"] = self.old_path + + def exec(self): + """ Run the code. Output to stdout and stderr is stored and + returned on a call to read + """ + if self.seconds_remaining <= 1: + print("Out of time. Aborted.", file=sys.stderr) + else: + with Watchdog(self.seconds_remaining): + try: + exec(self.run_code, self.scoped_globals) + except OutOfInput: + print("'input' function called when no input data available.", + file=sys.stderr) + except ExcessiveOutput: + print("Excessive output ... job aborted", + file=sys.stderr) + except Watchdog: + print("Time limit exceeded", file=sys.stderr) + # (mct63) Catch any invalid actions. + except InvalidAction as e: + print(f"Invalid Action: {e}", file=sys.stderr) + except Exception: + etype, value, tb = sys.exc_info() + tb_tuples = traceback.extract_tb(tb) + new_tb = [] + for filename, linenumber, scope, text in tb_tuples: + if filename == "": + new_tb.append(( + "__source.py", + linenumber, + scope, + self.run_code.splitlines()[linenumber - 1].strip() + )) + print("Traceback (most recent call last):", file=sys.stderr) + print(''.join(traceback.format_list(new_tb)), end='', file=sys.stderr) + print(traceback.format_exception_only(etype, value)[-1], end='', file=sys.stderr) + except SystemExit: + print("Unexpected termination: Please do not call exit() or quit().", + file=sys.stderr) + except KeyboardInterrupt: + print("KeyboardInterrupt", file=sys.stderr) + except GeneratorExit: + print("GeneratorExit", file=sys.stderr) + + # (mct63) Might as well catch the base exception in case something very strange happens. + # For example intentionally raiseing the BaseException to try and skip all of this. + except BaseException: + print("Caught BaseException. You did something very strange to get this message.", + file=sys.stderr) + + + + def read(self): + """ Get the output and error from the exec + """ + return sys.stdout.getvalue(), sys.stderr.getvalue() + + +class PyTask(languagetask.LanguageTask): + """A PyTask manages compiling (almost a NOP) and executing of a Python3 program. + """ + def __init__(self, params, code=None): + """Initialisation is delegated to the superclass. + """ + super().__init__(params, code) + self.executable_built = False + + def compile(self, make_executable=False): + """A No-op for Python. + """ + pass + + def discard_executable(self): + """A no-op for python""" + pass + + def run_code(self, standard_input=None): + """Run code using Aaron's CodeTrap + """ + sys.stdin = io.StringIO(standard_input) + with CodeTrap(self.code, self.params, floor(self.seconds_remaining())) as runner: + runner.exec() + output, error = runner.read() + self.stdout, self.stderr = output, error + return output, error + diff --git a/python3_scratchpad/__resulttable.py b/python3_scratchpad/__resulttable.py new file mode 100644 index 0000000..9d4b168 --- /dev/null +++ b/python3_scratchpad/__resulttable.py @@ -0,0 +1,250 @@ +"""Code for building and managing the result table for the tests. + The result table itself (the 'table' field of an object of this class) + is a list of lists of strings. The first row is the header row. + Columns are "Test", "Input" (optional), "Expected", "Got", "iscorrect", "ishidden" +""" +import html +import re +from collections import defaultdict + +MAX_STRING_LENGTH = 4000 # 4k is default maximum string length + + +class ResultTable: + def __init__(self, params): + self.params = params + self.mark = 0 + self.table = None + self.failed_hidden = False + self.aborted = False + self.has_stdins = False + self.has_tests = False + self.hiding = False + self.num_failed_tests = 0 + self.missing_tests = 0 + self.global_error = '' + self.column_formats = None + self.images = defaultdict(list) + default_params = { + 'stdinfromextra': False, + 'strictwhitespace': True, + 'floattolerance': None, + 'ALL_OR_NOTHING': True + } + for param, value in default_params.items(): + if param not in params: + self.params[param] = value + + + def set_header(self, testcases): + """Given the set of testcases, set the header as the first row of the result table + and set flags to indicate presence or absence + of various table columns. + """ + header = ['iscorrect'] + self.column_formats = ['%s'] + if any(test.testcode.strip() != '' for test in testcases): + header.append("Test") + self.has_tests = True + # If the test code should be rendered in html then set that as column format. + if any(getattr(test, 'test_code_html', None) for test in testcases): + self.column_formats.append('%h') + else: + self.column_formats.append('%s') + + stdins = [test.extra if self.params['stdinfromextra'] else test.stdin for test in testcases] + if any(stdin.rstrip() != '' for stdin in stdins): + header.append('Input') + self.column_formats.append('%s') + self.has_stdins = True + header += ['Expected', 'Got', 'iscorrect', 'ishidden'] + self.column_formats += ['%s', '%s', '%s', '%s'] + self.table = [header] + + def image_column_nums(self): + """A list of the numbers of columns containing images""" + return sorted(set([key[0] for key in self.images.keys()])) + + def get_column_formats(self): + """ An ordered list of the column formats. Columns containing images are forced into %h format. + Don't have formats for iscorrect and ishidden columns. + """ + image_columns = self.image_column_nums() + formats = [self.column_formats[i] if i not in image_columns else '%h' for i in range(len(self.column_formats))] + return formats[1:-2] + + def get_table(self): + """Return the current result table, with images added to appropriate cells. + Columns that contain images anywhere are converted to %h format and existing content in that column + is html-escaped, newlines replaced with
and wrapped in a div. + """ + result_table = [row[:] for row in self.table] # Clone the result table + + # Htmlise all columns containing images + for col_num in self.image_column_nums(): + for row_num in range(1, len(result_table)): + result_table[row_num][col_num] = self.htmlise(result_table[row_num][col_num]) + + # Append images + for ((col,row), image_list) in self.images.items(): + for image in image_list: + try: + result_table[row][col] += "
" + image + except IndexError: + pass # Testing must have aborted so discard image + + return result_table + + def reset(self): + if len(self.table) > 1: + del self.table[1:] + self.global_error = '' + self.num_failed_tests = self.mark = 0 + self.failed_hidden = self.hiding = self.aborted = False + + def tests_missed(self, num): + """Record the fact that we're missing some test results (timeout?)""" + self.missing_tests = num + + def record_global_error(self, error_message): + """Record some sort of global failure""" + self.global_error = error_message + + def add_row(self, testcase, result, error=''): + """Add a result row to the table for the given test and result""" + is_correct = self.check_correctness(result + error, testcase.expected) + row = [is_correct] + if self.has_tests: + if getattr(testcase, 'test_code_html', None): + row.append(testcase.test_code_html) + else: + row.append(testcase.testcode) + if self.has_stdins: + row.append(testcase.extra if self.params['stdinfromextra'] else testcase.stdin) + row.append(testcase.expected.rstrip()) + max_len = self.params.get('maxstringlength', MAX_STRING_LENGTH) + result = sanitise(result.rstrip('\n'), max_len) + + if error: + error_message = '*** RUN TIME ERROR(S) ***\n' + sanitise(error, max_len) + if result: + result = result + '\n' + error_message + else: + result = error_message + row.append(result) + + if is_correct: + self.mark += testcase.mark + else: + self.num_failed_tests += 1 + row.append(is_correct) + display = testcase.display.upper() + is_hidden = ( + self.hiding or + display == 'HIDE' or + (display == 'HIDE_IF_SUCCEED' and is_correct) or + (display == 'HIDE_IF_FAIL' and not is_correct) + ) + row.append(is_hidden) + if not is_correct and is_hidden: + self.failed_hidden = True + if not is_correct and testcase.hiderestiffail: + self.hiding = True + self.table.append(row) + if error: + self.aborted = True + + def get_mark(self): + return self.mark if self.num_failed_tests == 0 or not self.params['ALL_OR_NOTHING'] else 0 + + @staticmethod + def htmlise(s): + """Convert the given string to html by escaping '<' and '>'. + Wrap the whole lot in a div tag so the diff checker processes the whole table cell, + and within that a pre tag for correct laylout. + """ + return '
' + html.escape(s) + '
' + + def add_image(self, image_html, column_name, row_num): + """Store the given html_image for later inclusion in the cell at the given row and given column. + column_name is the name used for the column in the first (header) row. + row_num is the row number (0 origin, not including the header row). + """ + column_num = self.table[0].index(column_name) + self.images[column_num, row_num + 1].append(image_html) + + def equal_strings(self, s1, s2): + """ Compare the two strings s1 and s2 (expected and got respectively) + for equality, with regard to the template parameters + strictwhitespace and floattolerance. + """ + s1 = s1.rstrip() + s2 = s2.rstrip() + if not self.params['strictwhitespace']: + # Collapse white space if strict whitespace is not enforced + s1 = re.sub(r'\s+', ' ', s1) + s2 = re.sub(r'\s+', ' ', s2) + if self.params['floattolerance'] is None: + return s1 == s2 + else: + # Matching with a floating point tolerance. + # Use float pattern from Markus Schmassmann at + # https://stackoverflow.com/questions/12643009/regular-expression-for-floating-point-numbers + # except we don't match inf or nan which can be embedded in text strings. + tol = float(self.params['floattolerance']) + float_pat = r'([-+]?(?:(?:(?:[0-9]+[.]?[0-9]*|[.][0-9]+)(?:[ed][-+]?[0-9]+)?)))' + s1_bits = re.split(float_pat, s1) + s2_bits = re.split(float_pat, s2) + if len(s1_bits) != len(s2_bits): + return False + match = True + for bit1, bit2 in zip(s1_bits, s2_bits): + bit1 = bit1.strip() + bit2 = bit2.strip() + try: + f1 = float(bit1) + f2 = float(bit2) + if abs(f1 - f2) > tol * 1.001: # Allow tolerance on the float tolerance! + match = False + except ValueError: + if bit1 != bit2: + match = False + return match + + def check_correctness(self, expected, got): + """True iff expected matches got with relaxed white space requirements. + Additionally, if the template parameter floattolerance is set and is + non-zero, the two strings will be split by a floating-point literal + pattern and the floating-point bits will be matched to within the + given absolute tolerance. + """ + expected_lines = expected.rstrip().splitlines() + got_lines = got.rstrip().splitlines() + if len(got_lines) != len(expected_lines): + return False + else: + for exp, got in zip(expected_lines, got_lines): + if not self.equal_strings(exp, got): + return False + return True + + +def sanitise(s, max_len=MAX_STRING_LENGTH): + """Replace non-printing chars with escape sequences, right-strip. + Limit s to max_len by snipping out bits in the middle. + """ + result = '' + if len(s) > max_len: + s = s[0: max_len // 2] + "\n*** ***\n" + s[-max_len // 2:] + lines = s.rstrip().splitlines() + for line in lines: + for c in line.rstrip() + '\n': + if c < ' ' and c != '\n': + if c == '\t': + c = r'\t' + elif c == '\r': + c = r'\r' + else: + c = r'\{:03o}'.format(ord(c)) + result += c + return result.rstrip() diff --git a/python3_scratchpad/__tester.py b/python3_scratchpad/__tester.py new file mode 100644 index 0000000..82e98f1 --- /dev/null +++ b/python3_scratchpad/__tester.py @@ -0,0 +1,347 @@ +"""The generic (multi-language) main testing class that does all the work - trial compile, style checks, + run and grade. +""" +from __resulttable import ResultTable +import html +import os +import re +import __languagetask as languagetask +import base64 + + +# Values of QUESTION.precheck field +PRECHECK_DISABLED = 0 +PRECHECK_EMPTY = 1 +PRECHECK_EXAMPLES = 2 +PRECHECK_SELECTED = 3 +PRECHECK_ALL = 4 + +# Values of testtype +TYPE_NORMAL = 0 +TYPE_PRECHECKONLY = 1 +TYPE_BOTH = 2 + +# Global message for when a test-suite timeout occurs +TIMEOUT_MESSAGE = """A timeout occurred when running the whole test suite as a single program. +This is usually due to an endless loop in your code but can also arise if your code is very inefficient +and the accumulated time over all tests is excessive. Please ask a tutor or your lecturer if you need help +with making your program more efficient.""" + + +def get_jpeg_b64(filename): + """Return the contents of the given file (assumed to be jpeg) as a base64 + encoded string in utf-8. + """ + with open(filename, 'br') as fin: + contents = fin.read() + + return base64.b64encode(contents).decode('utf8') + + +class Tester: + def __init__(self, params, testcases): + """Initialise the instance, given the test of template and global parameters plus + all the testcases. Parameters required by this base class and all subclasses are: + 'STUDENT_ANSWER': code submitted by the student + 'SEPARATOR': the string to be used to separate tests in the output + 'ALL_OR_NOTHING: true if grading is all-or-nothing + 'stdinfromextra': true if the test-case 'extra' field is to be used for + standard input rather than the usual stdin field + 'runtestssingly': true to force a separate run for each test case + 'stdinfromextra': true if the extra field is used for standard input (legacy use only) + 'testisbash': true if tests are bash command line(s) rather than the default direct execution + of the compiled program. This can be used to supply command line arguments. + + """ + self.student_answer = self.clean(params['STUDENT_ANSWER']) + self.separator = params['SEPARATOR'] + self.all_or_nothing = params['ALL_OR_NOTHING'] + self.params = params + self.testcases = self.filter_tests(testcases) + self.result_table = ResultTable(params) + self.result_table.set_header(self.testcases) + + # It is assumed that in general subclasses will prefix student code by a prelude and + # postfix it by a postlude. + self.prelude = '' + self.prelude_length = 0 + self.postlude = '' + + self.task = None # SUBCLASS MUST DEFINE THIS + + def filter_tests(self, testcases): + """Return the relevant subset of the question's testcases. + This will be all testcases not marked precheck-only if it's not a precheck or all testcases if it is a + precheck and the question precheck is set to "All", or the appropriate subset in all other cases. + """ + if not self.params['IS_PRECHECK']: + return [test for test in testcases if test.testtype != TYPE_PRECHECKONLY] + elif self.params['QUESTION_PRECHECK'] == PRECHECK_ALL: + return testcases + elif self.params['QUESTION_PRECHECK'] == PRECHECK_EMPTY: + return [] + elif self.params['QUESTION_PRECHECK'] == PRECHECK_EXAMPLES: + return [test for test in testcases if test.useasexample] + elif self.params['QUESTION_PRECHECK'] == PRECHECK_SELECTED: + return [test for test in testcases if test.testtype in [TYPE_PRECHECKONLY, TYPE_BOTH]] + + def style_errors(self): + """Return a list of all the style errors. Implementation is language dependent. + Default is no style checking. + """ + return [] + + def single_program_build_possible(self): + """Return true if and only if the current configuration permits a single program to be + built and tried containing all tests. It should be true for write-a-program questions and + conditionally true for other types of questions that allow a "combinator" approach, + dependent on the presence of stdins in tests and other such conditions. + """ + raise NotImplementedError("Tester must have a single_program_build_possible method") + + def adjust_error_line_nums(self, error): + """Given a runtime error message, adjust it as may be required by the + language, e.g. adjusting line numbers + """ + raise NotImplementedError("Tester must have an adjust_error_line_nums method") + + def single_run_possible(self): + """Return true if a single program has been built and it is possible to use that in a single run + with all tests. + """ + return (self.task.executable_built + and not self.params['runtestssingly'] + and not self.result_table.has_stdins + and not self.params['testisbash']) + + def make_test_postlude(self, testcases): + """Return the postlude testing code containing all the testcode from + the given list of testcases (which may be the full set or a singleton list). + A separator must be printed between testcase outputs.""" + raise NotImplementedError("Tester must have a make_test_postlude method") + + def trial_compile(self): + """This function is the first check on the syntactic correctness of the submitted code. + It is called before any style checks are done. For compiled languages it should generally + call the standard language compiler on the student submitted code with any required prelude + added and, if possible, all tests included. CompileError should be raised if the compile fails, + which will abort all further testing. + If possible a complete ready-to-run executable should be built as well; if this succeeds, the + LanguageTasks 'executable_built' attribute should be set. This should be possible for write-a-program + questions or for write-a-function questions when there is no stdin data in any of the tests. + + Interpreted languages should perform what syntax checks are possible using the standard language tools. + If those checks succeeded, they should also attempt to construct a source program that incorporates all + the different tests (the old "combinator" approach) and ensure the task's 'executable_built' attribute + is True. + + The following implementation is sufficient for standard compiled languages like C, C++, Java. It + may need overriding for other languages. + """ + if self.single_program_build_possible(): + self.setup_for_test_runs(self.testcases) + make_executable = True + else: + self.postlude = '' + self.task.set_code(self.prelude + self.student_answer, self.prelude_length) + make_executable = False + + self.task.compile(make_executable) # Could raise CompileError + + def setup_for_test_runs(self, tests): + """Set the code and prelude length as appropriate for a run with all the given tests. May be called with + just a singleton list for tests if single_program_build_possible has returned false or if testing with + multiple tests has given exceptions. + This implementation may need to be overridden, e.g. if the student code should follow the test code, as + say in Matlab scripts. + """ + self.postlude = self.make_test_postlude(tests) + self.task.set_code(self.prelude + self.student_answer + self.postlude, self.prelude_length) + + def run_all_tests(self): + """Run all the tests, leaving self.ResultTable object containing all test results. + Can raise CompileError or RunError if things break. + If any runtime errors occur on the full test, drop back to running tests singly. + """ + done = False + if self.single_run_possible(): + # We have an executable ready to go, with no stdins or other show stoppers + output, error = self.task.run_code() + output = output.rstrip() + '\n' + error = error.strip() + '\n' + + # Generate a result table using all available test data. + results = output.split(self.separator + '\n') + errors = error.split(self.separator + '\n') + if len(results) == len(errors): + merged_results = [] + for result, error in zip(results, errors): + result = result.rstrip() + '\n' + if error.strip(): + adjusted_error = self.adjust_error_line_nums(error.rstrip()) + result += '\n*** RUN ERROR ***\n' + adjusted_error + merged_results.append(result) + + missed_tests = len(self.testcases) - len(merged_results) + + for test, output in zip(self.testcases, merged_results): + self.result_table.add_row(test, output) + + self.result_table.tests_missed(missed_tests) + if self.task.timed_out: + self.result_table.record_global_error(TIMEOUT_MESSAGE) + done = True + + if not done: + # Something broke. We will need to run each test case separately + self.task.executable_built = False + self.result_table.reset() + + if not done: + # If a single run isn't appropriate, do a separate run for each test case. + build_each_test = not self.task.executable_built + for i_test, test in enumerate(self.testcases): + if build_each_test: + self.setup_for_test_runs([test]) + self.task.compile(True) + standard_input = test.extra if self.params['stdinfromextra'] else test.stdin + if self.params['testisbash']: + output, error = self.task.run_code(standard_input, test.testcode) + else: + output, error = self.task.run_code(standard_input) + adjusted_error = self.adjust_error_line_nums(error.rstrip()) + self.result_table.add_row(test, output, adjusted_error) + if error and self.params['abortonerror']: + self.result_table.tests_missed(len(self.testcases) - i_test - 1) + break + + def compile_and_run(self): + """Phase one of the test operation: do a trial compile and then, if all is well and it's not a precheck, + continue on to run all tests. + Return a tuple mark, errors where mark is a fraction in 0 - 1 and errors is a list of all the errors. + self.test_results contains all the test details. + """ + mark = 0 + errors = [] + + # Do a trial compile, then a style check. If all is well, run the code + try: + self.trial_compile() + + if not self.params['nostylechecks']: + errors = self.style_errors() + if not errors: + if self.params['IS_PRECHECK'] and self.params['QUESTION_PRECHECK'] <= 1: + mark = 1 + else: + self.run_all_tests() + max_mark = sum(test.mark for test in self.testcases) + mark = self.result_table.get_mark() / max_mark # Fractional mark 0 - 1 + except languagetask.CompileError as err: + adjusted_error = self.adjust_error_line_nums(str(err).rstrip()) + errors.append("COMPILE ERROR\n" + adjusted_error) + except languagetask.RunError as err: + adjusted_error = self.adjust_error_line_nums(str(err).rstrip()) + errors.append('RUN ERROR\n' + adjusted_error) + return mark, errors + + def prerun_hook(self): + """A hook for subclasses to do initial setup or code hacks etc + Returns a list of errors, to which other errors are appended + """ + return [] + + def get_all_images_html(self): + """Search the current directory for images named _image.*(Expected|Got)(\d+).png. + For each such file construct an html img element with the data encoded + in a dataurl. + If we're running the sample answer, always return [] - images will be + picked up when we run the actual answer. + Returns a list of tuples (img_elements, column_name, row_number) where + column_name is either 'Expected' or 'Got', defining in which result table + column the image belongs and row number is the row (0-origin, excluding + the header row). + """ + images = [] + if self.params.get('running_sample_answer', False): + return [] + if self.params['imagewidth'] is not None: + width_spec = " width={}".format(self.params['imagewidth']) + else: + width_spec = "" + files = sorted(os.listdir('.')) + for filename in files: + match = re.match(r'_image[^.]*\.(Expected|Got)\.(\d+).png', filename) + if match: + image_data = get_jpeg_b64(filename) + img_template = '' + img_html = img_template.format(width_spec, image_data) + column = match.group(1) # Name of column + row = int(match.group(2)) # 0-origin row number + images.append((img_html, column, row)) + return images + + def test_code(self): + """The "main program" for testing. Returns the test outcome, ready to be printed by json.dumps""" + errors = self.prerun_hook() + if errors: + mark = 0 + else: + mark, errors = self.compile_and_run() + + outcome = {"fraction": mark} + + error_text = '\n'.join(errors) + # TODO - check if error line numbers are still being corrected in C and matlab + if self.params['IS_PRECHECK']: + if mark == 1: + prologue = "

Passed 🙂

" + else: + prologue = "

Failed, as follows.

" + elif errors: + prologue = "

Pre-run checks failed

\n" + else: + prologue = "" + + if prologue: + outcome['prologuehtml'] = prologue + self.htmlize(error_text) + + epilogue = '' + images = self.get_all_images_html() + if images: + for (image, column, row) in images: + self.result_table.add_image(image, column, row) + outcome['columnformats'] = self.result_table.get_column_formats() + + if len(self.result_table.table) > 1: + outcome['testresults'] = self.result_table.get_table() + outcome['showdifferences'] = True + + if self.result_table.global_error: + epilogue += "

Run Error

{}
".format( + self.htmlize(self.result_table.global_error)) + + if self.result_table.aborted: + epilogue = outcome.get('epiloguehtml', '') + ( + "
Testing was aborted due to runtime errors.
") + + if self.result_table.missing_tests != 0: + template = "
{} tests not run due to previous errors.
" + epilogue += template.format(self.result_table.missing_tests) + + if self.result_table.failed_hidden: + epilogue += "
One or more hidden tests failed.
" + + if epilogue: + outcome['epiloguehtml'] = epilogue + return outcome + + @staticmethod + def clean(code): + """Return the given code with trailing white space stripped from each line""" + return '\n'.join([line.rstrip() for line in code.split('\n')]) + '\n' + + @staticmethod + def htmlize(message): + """An html version of the given error message""" + return '
' + html.escape(message) + '
' if message else '' diff --git a/python3_scratchpad/__watchdog.py b/python3_scratchpad/__watchdog.py new file mode 100644 index 0000000..b40ae7a --- /dev/null +++ b/python3_scratchpad/__watchdog.py @@ -0,0 +1,27 @@ +#!/usr/bin/python +# file: watchdog.py +# license: MIT License +# From https://dzone.com/articles/simple-python-watchdog-timer + +import signal + +class Watchdog(Exception): + def __init__(self, time): + """Set up a timer alarm to go off in 'time' secs.""" + self.time = time + + def __enter__(self): + """Called on entering a 'with' block""" + signal.signal(signal.SIGALRM, self.handler) + signal.alarm(self.time) + + def __exit__(self, type, value, traceback): + """Exiting the with block. Cancel the watchdog""" + signal.alarm(0) + + def handler(self, signum, frame): + """Alarm went off. Raise Watchdog exception""" + raise self + + def __str__(self): + return "Watchdog timer expired after {} secs".format(self.time) diff --git a/python3_scratchpad/pytester.py b/python3_scratchpad/pytester.py new file mode 100644 index 0000000..9273e4d --- /dev/null +++ b/python3_scratchpad/pytester.py @@ -0,0 +1,255 @@ +"""The main python-program testing class that does all the work - style checks, + run and grade. A subclass of the generic tester. + Since each test can by run within the current instance of Python using + an exec, we avoid the usual complication of combinators by running + each test separately regardless of presence of stdin, testcode, etc. +""" +import __pytask as pytask +import re +from __tester import Tester +from __pystylechecker import StyleChecker + + +class PyTester(Tester): + def __init__(self, params, testcases): + """Initialise the instance, given the test of template and global parameters plus + all the testcases. Parameters relevant to this class are all those listed for the Tester class plus + 'extra' which takes the values 'pretest' or 'posttest' (the other possible value, 'stdin', has been + handled by the main template). + Additionally the support classes like stylechecker and pyparser need their + own params - q.v. + """ + super().__init__(params, testcases) # Most of the task is handed by the generic tester + + # Py-dependent attributes + self.task = pytask.PyTask(params) + self.prelude = '' + + if params['isfunction']: + if not self.has_docstring(): + self.prelude = '"""Dummy docstring for a function"""\n' + + if params['usesmatplotlib']: + self.prelude += '\n'.join([ + 'import os', + 'import matplotlib as _mpl', + '_mpl.use("Agg")', + 'from __plottools import print_plot_info', + ]) + '\n' + self.params['pylintoptions'].append("--disable=ungrouped-imports") + + if params['usesnumpy']: + self.prelude += 'import numpy as np\n' + self.params['pylintoptions'].append("--disable=unused-import,ungrouped-imports") + self.params['pylintoptions'].append("--extension-pkg-whitelist=numpy") + + for import_string in params['imports']: + if ' ' not in import_string: + self.prelude += 'import ' + import_string + '\n' + else: + self.prelude += import_string + '\n' + + if params['prelude'] != '': + self.prelude += '\n' + params['prelude'].rstrip() + '\n' + + try: + with open('_prefix.py') as prefix: + prefix_code = prefix.read() + self.prelude += prefix_code.rstrip() + '\n' + + except FileNotFoundError: + pass + + self.prelude_length = len(self.prelude.splitlines()) + if self.has_docstring() and self.prelude_length > 0: + # If we insert prelude in front of the docstring, pylint will + # give a missing docstring error. Our horrible hack solution is + # to insert an extra docstring at the start and turn off the + # resulting 'string statement has no effect' error. + self.prelude = '"""Dummy docstring for a function"""\n' + self.prelude + self.prelude_length += 1 + self.params['pylintoptions'].append("--disable=W0105") + self.style_checker = StyleChecker(self.prelude, self.params['STUDENT_ANSWER'], self.params) + + def has_docstring(self): + """True if the student answer has a docstring, which means that, + when stripped, it starts with a string literal. + """ + prog = self.params['STUDENT_ANSWER'].lstrip() + return prog.startswith('"') or prog.startswith("'") + + def style_errors(self): + """Return a list of all the style errors. Start with local tests and continue with pylint + only if there are no local errors. + """ + errors = [] + if self.params.get('localprechecks', True): + try: + errors += self.style_checker.local_errors() # Note: prelude not included so don't adjust line nums + except Exception as e: + errors += [str(e)] + else: + check_for_passive = (self.params['warnifpassiveoutput'] and self.params['isfunction']) + if check_for_passive and self.passive_output(): + errors.append("Your code was not expected to generate any output " + + "when executed stand-alone.\nDid you accidentally include " + + "your test code?") + + if len(errors) == 0 or self.params.get('forcepylint', False): + # Run precheckers (pylint, mypy) + try: + # Style-check the program without any test cases or other postlude added + errors += self.style_checker.style_errors() + except Exception as e: + error_text = '*** Unexpected error while running precheckers. Please report ***\n' + str(e) + errors += [error_text] + errors = [self.simplify_error(self.adjust_error_line_nums(error)) for error in errors] + errors = [error for error in errors if not error.startswith('************* Module')] + + errors = [error.replace(', ', '') for error in errors] # Another error tidying operation + if errors: + errors.append("\nSorry, but your code doesn't pass the style checks.") + return errors + + def prerun_hook(self): + """A hook for subclasses to do initial setup or code hacks etc + Returns a list of errors, to which other errors are appended. + In this class we use it to perform required hacks to disable + calls to main. If the call to main_hacks fails, assume the code + is bad and will get flagged by pylint in due course. + """ + try: + return self.main_hacks() + except: + return [] + + def passive_output(self): + """ Return the passive output from the student answer code + This is essentially a "dry run" of the code. + """ + code = self.prelude + self.params['STUDENT_ANSWER'] + if self.params['usesmatplotlib']: + code += '\n'.join([ + 'figs = _mpl.pyplot.get_fignums()', + 'if figs:', + ' print(f"{len(figs)} figures found")' + ]) + '\n' + task = pytask.PyTask(self.params, code) + task.compile() + captured_output, captured_error = task.run_code() + return (captured_output + '\n' + captured_error).strip() + + def make_test_postlude(self, testcases): + """Return the code that follows the student answer containing all the testcode + from the given list of testcases, which should always be of length 1 + (because we don't bother trying to combine all the tests into a + single run in Python) + """ + assert len(testcases) == 1 + if self.params['notest']: + return '' + test = testcases[0] + tester = '' + if self.params['globalextra'] and self.params['globalextra'] == 'pretest': + tester += self.params['GLOBAL_EXTRA'] + '\n' + if test.extra and self.params['extra'] == 'pretest': + tester += test.extra + '\n' + if test.testcode: + tester += test.testcode.rstrip() + '\n' + if self.params['globalextra'] and self.params['globalextra'] == 'posttest': + tester += self.params['GLOBAL_EXTRA'] + '\n' + if test.extra and self.params['extra'] == 'posttest': + tester += test.extra + '\n' + + if self.params['usesmatplotlib']: + if 'dpi' in self.params and self.params['dpi']: + extra = f", dpi={self.params['dpi']}" + else: + extra = '' + if self.params.get('running_sample_answer', False): + column = 'Expected' + else: + column = 'Got' + test_num = len(self.result_table.table) - 1 # 0-origin test number from result table + tester += '\n'.join([ + 'figs = _mpl.pyplot.get_fignums()', + 'for fig in figs:', + ' _mpl.pyplot.figure(fig)', + ' row = {}'.format(test_num), + ' column = "{}"'.format(column), + ' _mpl.pyplot.savefig("_image{}.{}.{}.png".format(fig, column, row), bbox_inches="tight"' + '{})'.format(extra), + ' _mpl.pyplot.close(fig)' + ]) + '\n' + return tester + + def single_program_build_possible(self): + """We avoid all the complication of trying to run all tests in + a single subprocess run by using exec to run each test singly. + """ + return False + + def adjust_error_line_nums(self, error): + """Subtract the prelude length of all line numbers in the given error message + """ + error_patterns = [ + (r'(.*.* \(syntax-error\).*)', []), + (r'(.*File ".*", line +)(\d+)(, in .*)', [2]), + (r'(.*: *)(\d+)(, *\d+:.*\(.*line +)(\d+)(\).*)', [2, 4]), + (r'(.*: *)(\d+)(, *\d+:.*\(.*\).*)', [2]), + (r'(.*:)(\d+)(:\d+: [A-Z]\d+: .*line )(\d+)(.*)', [2, 4]), + (r'(.*:)(\d+)(:\d+: [A-Z]\d+: .*)', [2]), + ] + output_lines = [] + for line in error.splitlines(): + for pattern, line_group_nums in error_patterns: + match = re.match(pattern, line) + if match: + line = '' + for i, group in enumerate(match.groups(), 1): + if i in line_group_nums: + linenum = int(match.group(i)) + adjusted = linenum - self.prelude_length + line += str(adjusted) + else: + line += group + break + + output_lines.append(line) + return '\n'.join(output_lines) + + def simplify_error(self, error): + """Return a simplified version of a pylint error with Line inserted in + lieu of __source.py:

: Xnnnn + """ + pattern = f'__source.py:(\d+): *\d+: *[A-Z]\d+: (.*)' + match = re.match(pattern, error) + if match: + return f"Line {match.group(1)}: {match.group(2)}" + else: + return error + + def main_hacks(self): + """Modify the code to be tested if params stripmain or stripmainifpresent' + are specified. Returns a list of errors encountered while so doing. + """ + errors = [] + if self.params['stripmain'] or self.params['stripmainifpresent']: + main_calls = self.style_checker.find_function_calls('main') + if self.params['stripmain'] and main_calls == []: + errors.append("No call to main() found") + else: + student_lines = self.student_answer.split('\n') + for (line, depth) in main_calls: + if depth == 0: + main_call = student_lines[line] + if not re.match(' *main\(\)', main_call): + errors.append(f"Illegal call to main().\n" + + "main should not take any parameters and should not return anything.") + else: + student_lines[line] = main_call.replace( + "main", "pass # Disabled call to main") + else: + student_lines[line] += " # We've let you call main here." + self.params['STUDENT_ANSWER'] = self.student_answer = '\n'.join(student_lines) + '\n' + + return errors