From e60464dbbbb39f5c19a93019342b270c277a7d93 Mon Sep 17 00:00:00 2001 From: Austin Cory Bart Date: Sat, 27 Jul 2024 12:31:13 -0400 Subject: [PATCH] Started improving some of the docs for the command line interface --- docsrc/_static/argparse.css | 6 +++ docsrc/conf.py | 5 +++ docsrc/index.rst | 1 + docsrc/teachers/cli.rst | 57 +++++++++++++++++++++++++ pedal/command_line/command_line.py | 16 ++++--- pedal/command_line/modes.py | 68 +++++++++++++++++++++++++++--- 6 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 docsrc/_static/argparse.css create mode 100644 docsrc/teachers/cli.rst diff --git a/docsrc/_static/argparse.css b/docsrc/_static/argparse.css new file mode 100644 index 00000000..dda6b290 --- /dev/null +++ b/docsrc/_static/argparse.css @@ -0,0 +1,6 @@ +.wy-table-responsive table td { + white-space: normal !important; +} +.wy-table-responsive { + overflow: visible !important; +} \ No newline at end of file diff --git a/docsrc/conf.py b/docsrc/conf.py index 114e5a51..be3e6509 100644 --- a/docsrc/conf.py +++ b/docsrc/conf.py @@ -46,6 +46,7 @@ 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', + 'sphinxarg.ext', 'feedback_function_directive'] # Add any paths that contain templates here, relative to this directory. @@ -110,6 +111,10 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +html_css_files = [ + 'argparse.css' +] + # Custom sidebar templates, must be a dictionary that maps document names # to template names. # diff --git a/docsrc/index.rst b/docsrc/index.rst index a3e418bf..e7205004 100644 --- a/docsrc/index.rst +++ b/docsrc/index.rst @@ -30,6 +30,7 @@ rather than an afterthought. teachers/quickstart teachers/examples teachers/reference + teachers/cli teachers/integrations developers/ffs developers/api diff --git a/docsrc/teachers/cli.rst b/docsrc/teachers/cli.rst new file mode 100644 index 00000000..5198895d --- /dev/null +++ b/docsrc/teachers/cli.rst @@ -0,0 +1,57 @@ +.. _cli: + +Command-Line Interface +====================== + +In addition to being a library, Pedal also provides a command-line interface (CLI) for analyzing student code. +This is useful for batch processing or on individual submissions (e.g., through a platform like GradeScope or VSCodeEdu). +For specific integrations, see the :ref:`integrations` section. + +Command Line Modes +------------------ + +.. autoclass:: pedal.command_line.modes.SandboxPipeline + +.. autoclass:: pedal.command_line.modes.RunPipeline + +.. autoclass:: pedal.command_line.modes.FeedbackPipeline + +.. autoclass:: pedal.command_line.modes.GradePipeline + +.. autoclass:: pedal.command_line.modes.StatsPipeline + +.. autoclass:: pedal.command_line.modes.VerifyPipeline + +.. autoclass:: pedal.command_line.modes.DebugPipeline + +File Formats +------------ + +Pedal can read and write several file formats. The most common are: + +- Python files (``.py``) +- Folders containing Python files +- Archive files (``.zip``, ``.tar.gz``, etc.) containing Python files +- JSON files (``.json``) that have Python code embedded in them +- CSV files (``.csv``) that contain Python code + +Pedal can also work with ProgSnap data dumps, containing code snapshots: +- Zip files (``.zip``) +- CSV files (``.csv``) +- Sqlite files (``.db``) - this is often the fastest file format! + +By far, the simplest option is just Python files or folders containing Python files. +However, the other formats can be useful for more complex scenarios with additional metadata. + +Pedal Job Files (``.pedal``) can be used to store the configuration for a Pedal job. + +Command Line Parameters +----------------------- + +The CLI is invoked with the ``pedal`` command. It accepts the following parameters: + + +.. argparse:: + :module: pedal.command_line.command_line + :func: build_parser + :prog: pedal diff --git a/pedal/command_line/command_line.py b/pedal/command_line/command_line.py index 00ca6f12..91873fa1 100644 --- a/pedal/command_line/command_line.py +++ b/pedal/command_line/command_line.py @@ -69,11 +69,11 @@ def main(args=None): return pipeline(args).execute() -def parse_args(reduced_mode=False): - """ Parse the arguments passed into the command line. """ +def build_parser(reduced_mode=False): parser = argparse.ArgumentParser(description='Run instructor control ' 'script on student submissions.') - parser.add_argument('mode', help="What kind of Pedal analysis you're running", + parser.add_argument('mode', + help="What kind of Pedal analysis you're running. See the description of each mode above.", choices=list(MODES.PIPELINES)) if not reduced_mode: parser.add_argument('instructor', help='The path to the instructor control ' @@ -105,7 +105,7 @@ def parse_args(reduced_mode=False): " more friendly. If not given, then will default" " to the instructor filename.", default=None) parser.add_argument('--progsnap_profile', - default='blockpy', # TODO: Change this biased default + default='blockpy', # TODO: Change this biased default help="Uses the given profile's default settings for" " loading in a ProgSnap2 dataset", ) @@ -116,7 +116,8 @@ def parse_args(reduced_mode=False): ' submissions are run. Mostly for testing' ' purposes.', default=None) - parser.add_argument('--resolver', help='Choose a different resolver to use (the name of a function defined in the instructor control script).', + parser.add_argument('--resolver', + help='Choose a different resolver to use (the name of a function defined in the instructor control script).', default='resolve') parser.add_argument('--ics_direct', help="Give the instructor code directly" " instead of loading from a file.", @@ -155,6 +156,11 @@ def parse_args(reduced_mode=False): "running scripts in parallel.", choices=["threads", "processes", "none"]) ''' + return parser + +def parse_args(reduced_mode=False): + """ Parse the arguments passed into the command line. """ + parser = build_parser(reduced_mode) args = parser.parse_args() if args.instructor_name is None: args.instructor_name = args.instructor diff --git a/pedal/command_line/modes.py b/pedal/command_line/modes.py index 8b3ccd7d..964917dc 100644 --- a/pedal/command_line/modes.py +++ b/pedal/command_line/modes.py @@ -59,6 +59,11 @@ def get_python_files(paths): class BundleResult: + """ + Represents the result of running an instructor control script on a submission. + This includes not only the output and error, but also the resolution of the feedback (aka + the final feedback). Also includes the data that was generated during the execution. + """ def __init__(self, data, output, error, resolution): self.data = data self.output = output @@ -78,6 +83,12 @@ def to_json(self): ) class Bundle: + """ + Represents the combination of an instructor control script and a submission that it is + being run on. Also includes the environment that the script is being run in, and the + result of the execution (once the execution is finished). Finally, also includes the configuration + that was used to run the bundle. + """ def __init__(self, config, script, submission): self.config = config self.script = script @@ -118,8 +129,6 @@ def run_ics_bundle(self, resolver='resolve', skip_tifa=False, skip_run=False): grader_exec = compile(self.script, self.submission.instructor_file, 'exec') exec(grader_exec, global_data) - #print(repr(self.script), file=x) - #print(list(global_data.keys()), file=x) if 'MAIN_REPORT' in global_data: if not global_data['MAIN_REPORT'].resolves: if resolver in global_data: @@ -137,8 +146,11 @@ def run_ics_bundle(self, resolver='resolve', skip_tifa=False, skip_run=False): class AbstractPipeline: - """ Generic pipeline for handling all the phases of executing instructor - control scripts on submissions, and reformating the output. """ + """ + Generic pipeline for handling all the phases of executing instructor + control scripts on submissions, and reformating the output. + Should be subclassed instead of used directly. + """ def __init__(self, config): if isinstance(config, dict): @@ -318,6 +330,11 @@ def process_output(self): class FeedbackPipeline(AbstractPipeline): + """ + ``feedback``: Pipeline for running the instructor control script on a submission and + then printing the resolver output to the console. Often the most useful + if you are trying to deliver the feedback without a grade. + """ def process_output(self): for bundle in self.submissions: #print(bundle.submission.instructor_file, @@ -334,10 +351,16 @@ def process_output(self): class RunPipeline(AbstractPipeline): + """ + ``run``: Pipeline for running the instructor control script on a submission and generating a report + file in the `ini` file format. This is a simple file format that has a lot of the interesting + fields. The file is not actually dumped to the filesystem, but instead printed directly. + So this is a good way to run students' code in a sandbox and see what comes out. + """ def process_output(self): for bundle in self.submissions: - print(bundle.submission.instructor_file, - bundle.submission.main_file) + #print(bundle.submission.instructor_file, + # bundle.submission.main_file) if bundle.result.error: print(bundle.result.error) elif bundle.result.resolution: @@ -348,6 +371,11 @@ def process_output(self): class StatsPipeline(AbstractPipeline): + """ + ``stats``: Pipeline for running the instructor control script on a submission and then + dumping a JSON report with all the feedback objects. This is useful for + analyzing the feedback objects in a more programmatic way. + """ def run_control_scripts(self): for bundle in tqdm(self.submissions): bundle.run_ics_bundle(resolver='stats_resolve', skip_tifa=self.config.skip_tifa, @@ -380,6 +408,14 @@ def process_output(self): class VerifyPipeline(AbstractPipeline): + """ + ``verify``: Pipeline for running the instructor control script on a submission and then + comparing the output to an expected output file. This is useful for verifying + that the feedback is correct (at least, as correct as the expected output). + + You can also use this pipeline to generate the output files, to quickly create + regression "tests" of your feedback scripts. + """ def process_output(self): for bundle in self.submissions: bundle.run_ics_bundle(resolver=self.config.resolver, skip_tifa=self.config.skip_tifa, @@ -475,6 +511,13 @@ def run_cases(self): class GradePipeline(AbstractPipeline): + """ + ``grade``: Pipeline for running the instructor control script on a submission and then outputing + the grade to the console. This is useful for quickly grading a set of submissions. + The instructor file, student data, and assignment are also all printed out in the following CSV format: + + instructor_file, student_file, student_email, assignment_name, score, correct + """ def process_output(self): if self.config.output == 'stdout': self.print_bundles(sys.stdout) @@ -499,7 +542,12 @@ def print_bundles(self, target): class SandboxPipeline(AbstractPipeline): - """ Run the given script in a sandbox. """ + """ + ``sandbox``: Pipeline for running ONLY the student's code, and then outputing the results to the console. + There is no instructor control script logic, although the Source tool does check that the + student's code is syntactically correct. Otherwise, the students' code is run in a Sandbox mode. + This is useful if you just want to safely execute student code and observe their output. + """ ICS = """from pedal import * verify() @@ -546,6 +594,12 @@ def process_output(self): class DebugPipeline(AbstractPipeline): + """ + ``debug``: Pipeline for running the instructor control script on a submission and then outputing + the full results to the console. This is useful for debugging the instructor control + script, as it will show the full output, error, all of the feedback objects considered, + and the final feedback. + """ def process_output(self): for bundle in self.submissions: print(bundle.submission.instructor_file,