Started improving some of the docs for the command line interface

docsrc/_static/argparse.css

@@ -0,0 +1,6 @@
+.wy-table-responsive table td {
+    white-space: normal !important;
+}
+.wy-table-responsive {
+    overflow: visible !important;
+}

docsrc/conf.py

@@ -46,6 +46,7 @@
               'sphinx.ext.ifconfig',
               'sphinx.ext.viewcode',
               'sphinx.ext.githubpages',
+              'sphinxarg.ext',
               'feedback_function_directive']
 
 # Add any paths that contain templates here, relative to this directory.
@@ -110,6 +111,10 @@
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+html_css_files = [
+    'argparse.css'
+]
+
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
 #

docsrc/index.rst

@@ -30,6 +30,7 @@ rather than an afterthought.
     teachers/quickstart
     teachers/examples
     teachers/reference
+    teachers/cli
     teachers/integrations
     developers/ffs
     developers/api

docsrc/teachers/cli.rst

@@ -0,0 +1,57 @@
+.. _cli:
+
+Command-Line Interface
+======================
+
+In addition to being a library, Pedal also provides a command-line interface (CLI) for analyzing student code.
+This is useful for batch processing or on individual submissions (e.g., through a platform like GradeScope or VSCodeEdu).
+For specific integrations, see the :ref:`integrations` section.
+
+Command Line Modes
+------------------
+
+.. autoclass:: pedal.command_line.modes.SandboxPipeline
+
+.. autoclass:: pedal.command_line.modes.RunPipeline
+
+.. autoclass:: pedal.command_line.modes.FeedbackPipeline
+
+.. autoclass:: pedal.command_line.modes.GradePipeline
+
+.. autoclass:: pedal.command_line.modes.StatsPipeline
+
+.. autoclass:: pedal.command_line.modes.VerifyPipeline
+
+.. autoclass:: pedal.command_line.modes.DebugPipeline
+
+File Formats
+------------
+
+Pedal can read and write several file formats. The most common are:
+
+- Python files (``.py``)
+- Folders containing Python files
+- Archive files (``.zip``, ``.tar.gz``, etc.) containing Python files
+- JSON files (``.json``) that have Python code embedded in them
+- CSV files (``.csv``) that contain Python code
+
+Pedal can also work with ProgSnap data dumps, containing code snapshots:
+- Zip files (``.zip``)
+- CSV files (``.csv``)
+- Sqlite files (``.db``) - this is often the fastest file format!
+
+By far, the simplest option is just Python files or folders containing Python files.
+However, the other formats can be useful for more complex scenarios with additional metadata.
+
+Pedal Job Files (``.pedal``) can be used to store the configuration for a Pedal job.
+
+Command Line Parameters
+-----------------------
+
+The CLI is invoked with the ``pedal`` command. It accepts the following parameters:
+
+
+.. argparse::
+    :module: pedal.command_line.command_line
+    :func: build_parser
+    :prog: pedal

pedal/command_line/command_line.py

@@ -69,11 +69,11 @@ def main(args=None):
     return pipeline(args).execute()
 
 
-def parse_args(reduced_mode=False):
-    """ Parse the arguments passed into the command line. """
+def build_parser(reduced_mode=False):
     parser = argparse.ArgumentParser(description='Run instructor control '
                                                  'script on student submissions.')
-    parser.add_argument('mode', help="What kind of Pedal analysis you're running",
+    parser.add_argument('mode',
+                        help="What kind of Pedal analysis you're running. See the description of each mode above.",
                         choices=list(MODES.PIPELINES))
     if not reduced_mode:
         parser.add_argument('instructor', help='The path to the instructor control '
@@ -105,7 +105,7 @@ def parse_args(reduced_mode=False):
                              " more friendly. If not given, then will default"
                              " to the instructor filename.", default=None)
     parser.add_argument('--progsnap_profile',
-                        default='blockpy', # TODO: Change this biased default
+                        default='blockpy',  # TODO: Change this biased default
                         help="Uses the given profile's default settings for"
                              " loading in a ProgSnap2 dataset",
                         )
@@ -116,7 +116,8 @@ def parse_args(reduced_mode=False):
                                         ' submissions are run. Mostly for testing'
                                         ' purposes.',
                         default=None)
-    parser.add_argument('--resolver', help='Choose a different resolver to use (the name of a function defined in the instructor control script).',
+    parser.add_argument('--resolver',
+                        help='Choose a different resolver to use (the name of a function defined in the instructor control script).',
                         default='resolve')
     parser.add_argument('--ics_direct', help="Give the instructor code directly"
                                              " instead of loading from a file.",
@@ -155,6 +156,11 @@ def parse_args(reduced_mode=False):
                                                    "running scripts in parallel.",
                         choices=["threads", "processes", "none"])
     '''
+    return parser
+
+def parse_args(reduced_mode=False):
+    """ Parse the arguments passed into the command line. """
+    parser = build_parser(reduced_mode)
     args = parser.parse_args()
     if args.instructor_name is None:
         args.instructor_name = args.instructor

pedal/command_line/modes.py

@@ -59,6 +59,11 @@ def get_python_files(paths):
 
 
 class BundleResult:
+    """
+    Represents the result of running an instructor control script on a submission.
+    This includes not only the output and error, but also the resolution of the feedback (aka
+    the final feedback). Also includes the data that was generated during the execution.
+    """
     def __init__(self, data, output, error, resolution):
         self.data = data
         self.output = output
@@ -78,6 +83,12 @@ def to_json(self):
         )
 
 class Bundle:
+    """
+    Represents the combination of an instructor control script and a submission that it is
+    being run on. Also includes the environment that the script is being run in, and the
+    result of the execution (once the execution is finished). Finally, also includes the configuration
+    that was used to run the bundle.
+    """
     def __init__(self, config, script, submission):
         self.config = config
         self.script = script
@@ -118,8 +129,6 @@ def run_ics_bundle(self, resolver='resolve', skip_tifa=False, skip_run=False):
                     grader_exec = compile(self.script,
                                           self.submission.instructor_file, 'exec')
                     exec(grader_exec, global_data)
-                    #print(repr(self.script), file=x)
-                    #print(list(global_data.keys()), file=x)
                     if 'MAIN_REPORT' in global_data:
                         if not global_data['MAIN_REPORT'].resolves:
                             if resolver in global_data:
@@ -137,8 +146,11 @@ def run_ics_bundle(self, resolver='resolve', skip_tifa=False, skip_run=False):
 
 
 class AbstractPipeline:
-    """ Generic pipeline for handling all the phases of executing instructor
-    control scripts on submissions, and reformating the output. """
+    """
+    Generic pipeline for handling all the phases of executing instructor
+    control scripts on submissions, and reformating the output.
+    Should be subclassed instead of used directly.
+    """
 
     def __init__(self, config):
         if isinstance(config, dict):
@@ -318,6 +330,11 @@ def process_output(self):
 
 
 class FeedbackPipeline(AbstractPipeline):
+    """
+    ``feedback``: Pipeline for running the instructor control script on a submission and
+    then printing the resolver output to the console. Often the most useful
+    if you are trying to deliver the feedback without a grade.
+    """
     def process_output(self):
         for bundle in self.submissions:
             #print(bundle.submission.instructor_file,
@@ -334,10 +351,16 @@ def process_output(self):
 
 
 class RunPipeline(AbstractPipeline):
+    """
+    ``run``: Pipeline for running the instructor control script on a submission and generating a report
+    file in the `ini` file format. This is a simple file format that has a lot of the interesting
+    fields. The file is not actually dumped to the filesystem, but instead printed directly.
+    So this is a good way to run students' code in a sandbox and see what comes out.
+    """
     def process_output(self):
         for bundle in self.submissions:
-            print(bundle.submission.instructor_file,
-                  bundle.submission.main_file)
+            #print(bundle.submission.instructor_file,
+            #      bundle.submission.main_file)
             if bundle.result.error:
                 print(bundle.result.error)
             elif bundle.result.resolution:
@@ -348,6 +371,11 @@ def process_output(self):
 
 
 class StatsPipeline(AbstractPipeline):
+    """
+    ``stats``: Pipeline for running the instructor control script on a submission and then
+    dumping a JSON report with all the feedback objects. This is useful for
+    analyzing the feedback objects in a more programmatic way.
+    """
     def run_control_scripts(self):
         for bundle in tqdm(self.submissions):
             bundle.run_ics_bundle(resolver='stats_resolve', skip_tifa=self.config.skip_tifa,
@@ -380,6 +408,14 @@ def process_output(self):
 
 
 class VerifyPipeline(AbstractPipeline):
+    """
+    ``verify``: Pipeline for running the instructor control script on a submission and then
+    comparing the output to an expected output file. This is useful for verifying
+    that the feedback is correct (at least, as correct as the expected output).
+
+    You can also use this pipeline to generate the output files, to quickly create
+    regression "tests" of your feedback scripts.
+    """
     def process_output(self):
         for bundle in self.submissions:
             bundle.run_ics_bundle(resolver=self.config.resolver, skip_tifa=self.config.skip_tifa,
@@ -475,6 +511,13 @@ def run_cases(self):
 
 
 class GradePipeline(AbstractPipeline):
+    """
+    ``grade``: Pipeline for running the instructor control script on a submission and then outputing
+    the grade to the console. This is useful for quickly grading a set of submissions.
+    The instructor file, student data, and assignment are also all printed out in the following CSV format:
+
+    instructor_file, student_file, student_email, assignment_name, score, correct
+    """
     def process_output(self):
         if self.config.output == 'stdout':
             self.print_bundles(sys.stdout)
@@ -499,7 +542,12 @@ def print_bundles(self, target):
 
 
 class SandboxPipeline(AbstractPipeline):
-    """ Run the given script in a sandbox. """
+    """
+    ``sandbox``: Pipeline for running ONLY the student's code, and then outputing the results to the console.
+    There is no instructor control script logic, although the Source tool does check that the
+    student's code is syntactically correct. Otherwise, the students' code is run in a Sandbox mode.
+    This is useful if you just want to safely execute student code and observe their output.
+    """
 
     ICS = """from pedal import *
 verify()
@@ -546,6 +594,12 @@ def process_output(self):
 
 
 class DebugPipeline(AbstractPipeline):
+    """
+    ``debug``: Pipeline for running the instructor control script on a submission and then outputing
+    the full results to the console. This is useful for debugging the instructor control
+    script, as it will show the full output, error, all of the feedback objects considered,
+    and the final feedback.
+    """
     def process_output(self):
         for bundle in self.submissions:
             print(bundle.submission.instructor_file,