Merge pull request #249 from ucbds-infra/show-all-public

ucbds-infra · Jun 17, 2021 · cdba2dc · cdba2dc
2 parents a0bc5d0 + 90586af
commit cdba2dc
Show file tree

Hide file tree

Showing 12 changed files with 23 additions and 27 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -59,4 +59,4 @@ RUN mkdir /autograder
 ADD requirements.txt /tmp/requirements.txt
 RUN pip install -r /tmp/requirements.txt
 
-RUN pip install otter-grader==2.2.1
+RUN pip install git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -2,6 +2,10 @@
 
 **Unreleased:**
 
+* Added `show_all_public` configuration for grading
+
+**v2.2.1:**
+
 * Added metadata for each test case to Otter Assign format
 * Added success and failure messages for test cases
 * Added Gmail notifications plugin

diff --git a/otter/generate/templates/python/requirements.txt b/otter/generate/templates/python/requirements.txt
@@ -13,6 +13,6 @@ nbformat
 dill
 jupytext
 numpy
-otter-grader==2.2.1
+git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2
 {% endif %}{% if other_requirements %}
 {{ other_requirements }}{% endif %}
diff --git a/otter/generate/templates/r/requirements.txt b/otter/generate/templates/r/requirements.txt
@@ -14,4 +14,4 @@ dill
 rpy2
 jupytext
 numpy
-otter-grader==2.2.1
+git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2
diff --git a/otter/run/run_autograder/constants.py b/otter/run/run_autograder/constants.py
@@ -7,6 +7,7 @@
     "points_possible": None,
     "show_stdout": False,
     "show_hidden": False,
+    "show_all_public": False,
     "seed": None,
     "grade_from_log": False,
     "serialized_variables": {},

diff --git a/otter/test_files/__init__.py b/otter/test_files/__init__.py
@@ -49,21 +49,6 @@ def __init__(self, test_files):
         # self.results = {}
         self.output = None
         self.all_hidden = False
-
-        # total_score, points_possible = 0, 0
-        # for test_file in test_files:
-        #     for test_case_result in test_file.test_case_results:
-        #         case_pts = test_case_result.test_case.points
-        #         name = test_case_result.test_case.name
-        #         self.results[name] = GradingTestCaseResult(
-        #             name = test_case_result.test_case.name,
-        #             score = case_pts * test_case_result.passed,
-        #             possible = case_pts,
-        #             hidden = test_case_result.test_case.hidden,
-        #             incorrect = not test_case_result.passed,
-        #             test_case_result = test_case_result,
-        #             test_file = test_file,
-        #         )
 
     def __repr__(self):
         return pprint.pformat(self.to_dict(), indent=2)
@@ -258,9 +243,13 @@ def to_gradescope_dict(self, config):
             output["output"] = self.output
             # TODO: use output to display public test case results?
 
-        # hidden visibility determined by show_hidden_tests_on_release
+        # hidden visibility determined by show_hidden
         hidden_test_visibility = ("hidden", "after_published")[options["show_hidden"]]
 
+        # if show_all_public is true and all tests are public tests, display all tests in results
+        if options["show_all_public"] and all(tf.all_public for tf in self.results.values()):
+            hidden_test_visibility = "visible"
+
         # start w/ summary of public tests
         output["tests"].append({
             "name": "Public Tests",
@@ -270,22 +259,20 @@ def to_gradescope_dict(self, config):
 
         for test_name in self.test_files:
             test_file = self.get_result(test_name)
-            # hidden, incorrect = result.hidden, result.incorrect
             score, possible = test_file.score, test_file.possible
 
             output["tests"].append({
                 "name": test_file.name,
                 "score": score,
                 "max_score": possible,
-                "visibility": hidden_test_visibility, #  if hidden else 'visible',
+                "visibility": hidden_test_visibility,
                 "output": test_file.summary(),
             })
 
         if options["show_stdout"]:
             output["stdout_visibility"] = "after_published"
 
         if options["points_possible"] is not None:
-            # output["score"] = self.total / self.possible * options["points_possible"]
             try:
                 output["score"] = self.total / self.possible * options["points_possible"]
             except ZeroDivisionError:

diff --git a/otter/test_files/abstract_test.py b/otter/test_files/abstract_test.py
@@ -127,6 +127,10 @@ def passed_all(self):
     def passed_all_public(self):
         return all(tcr.passed for tcr in self.test_case_results if not tcr.test_case.hidden)
 
+    @property
+    def all_public(self):
+        return all(not tc.hidden for tc in self.test_cases)
+
     @property
     def grade(self):
         if self.all_or_nothing and not self.passed_all:

diff --git a/test/test-assign/gs-autograder-correct/requirements.txt b/test/test-assign/gs-autograder-correct/requirements.txt
@@ -13,4 +13,4 @@ nbformat
 dill
 jupytext
 numpy
-otter-grader==2.2.1
+git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2
diff --git a/test/test-assign/rmd-autograder-correct/requirements.txt b/test/test-assign/rmd-autograder-correct/requirements.txt
@@ -14,4 +14,4 @@ dill
 rpy2
 jupytext
 numpy
-otter-grader==2.2.1
+git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2
diff --git a/test/test-run/autograder/source/requirements.txt b/test/test-run/autograder/source/requirements.txt
@@ -13,6 +13,6 @@ nbformat
 dill
 jupytext
 numpy
-otter-grader==2.2.1
+git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2
 
 tqdm
diff --git a/test/test_generate/test-autograder/autograder-correct/requirements.txt b/test/test_generate/test-autograder/autograder-correct/requirements.txt
@@ -13,6 +13,6 @@ nbformat
 dill
 jupytext
 numpy
-otter-grader==2.2.1
+git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2
 
 tqdm
diff --git a/test/test_generate/test-autograder/autograder-custom-env/requirements.txt b/test/test_generate/test-autograder/autograder-custom-env/requirements.txt
@@ -13,6 +13,6 @@ nbformat
 dill
 jupytext
 numpy
-otter-grader==2.2.1
+git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2
 
 tqdm