From 54636f71f6f759bfa7593d11598b83927bc358a2 Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Tue, 15 Jun 2021 20:56:51 -0700 Subject: [PATCH 1/4] added show_all_public to configurations --- otter/run/run_autograder/constants.py | 1 + otter/test_files/__init__.py | 25 ++++++------------------- otter/test_files/abstract_test.py | 4 ++++ 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/otter/run/run_autograder/constants.py b/otter/run/run_autograder/constants.py index 982b056de..af3327540 100644 --- a/otter/run/run_autograder/constants.py +++ b/otter/run/run_autograder/constants.py @@ -7,6 +7,7 @@ "points_possible": None, "show_stdout": False, "show_hidden": False, + "show_all_public": False, "seed": None, "grade_from_log": False, "serialized_variables": {}, diff --git a/otter/test_files/__init__.py b/otter/test_files/__init__.py index 68e8cda56..9e42811e7 100644 --- a/otter/test_files/__init__.py +++ b/otter/test_files/__init__.py @@ -49,21 +49,6 @@ def __init__(self, test_files): # self.results = {} self.output = None self.all_hidden = False - - # total_score, points_possible = 0, 0 - # for test_file in test_files: - # for test_case_result in test_file.test_case_results: - # case_pts = test_case_result.test_case.points - # name = test_case_result.test_case.name - # self.results[name] = GradingTestCaseResult( - # name = test_case_result.test_case.name, - # score = case_pts * test_case_result.passed, - # possible = case_pts, - # hidden = test_case_result.test_case.hidden, - # incorrect = not test_case_result.passed, - # test_case_result = test_case_result, - # test_file = test_file, - # ) def __repr__(self): return pprint.pformat(self.to_dict(), indent=2) @@ -258,9 +243,13 @@ def to_gradescope_dict(self, config): output["output"] = self.output # TODO: use output to display public test case results? - # hidden visibility determined by show_hidden_tests_on_release + # hidden visibility determined by show_hidden hidden_test_visibility = ("hidden", "after_published")[options["show_hidden"]] + # if show_all_public is true and all tests are public tests, display all tests in results + if options["show_all_public"] and all(tf.all_public for tf in self.test_files.values()): + hidden_test_visibility = "visible" + # start w/ summary of public tests output["tests"].append({ "name": "Public Tests", @@ -270,14 +259,13 @@ def to_gradescope_dict(self, config): for test_name in self.test_files: test_file = self.get_result(test_name) - # hidden, incorrect = result.hidden, result.incorrect score, possible = test_file.score, test_file.possible output["tests"].append({ "name": test_file.name, "score": score, "max_score": possible, - "visibility": hidden_test_visibility, # if hidden else 'visible', + "visibility": hidden_test_visibility, "output": test_file.summary(), }) @@ -285,7 +273,6 @@ def to_gradescope_dict(self, config): output["stdout_visibility"] = "after_published" if options["points_possible"] is not None: - # output["score"] = self.total / self.possible * options["points_possible"] try: output["score"] = self.total / self.possible * options["points_possible"] except ZeroDivisionError: diff --git a/otter/test_files/abstract_test.py b/otter/test_files/abstract_test.py index 117c87edf..6c2368843 100644 --- a/otter/test_files/abstract_test.py +++ b/otter/test_files/abstract_test.py @@ -127,6 +127,10 @@ def passed_all(self): def passed_all_public(self): return all(tcr.passed for tcr in self.test_case_results if not tcr.test_case.hidden) + @property + def all_public(self): + return all(not tc.hidden for tc in self.test_cases) + @property def grade(self): if self.all_or_nothing and not self.passed_all: From ee1e893e65f1fba759d7d9da5f6581924dcfdaf8 Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Tue, 15 Jun 2021 20:57:09 -0700 Subject: [PATCH 2/4] release 54636f71f6f759bfa7593d11598b83927bc358a2 --- Dockerfile | 2 +- otter/generate/templates/python/requirements.txt | 2 +- otter/generate/templates/r/requirements.txt | 2 +- test/test-assign/gs-autograder-correct/requirements.txt | 2 +- test/test-assign/rmd-autograder-correct/requirements.txt | 2 +- test/test-run/autograder/source/requirements.txt | 2 +- .../test-autograder/autograder-correct/requirements.txt | 2 +- .../test-autograder/autograder-custom-env/requirements.txt | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4828f76b4..10efae5f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,4 +59,4 @@ RUN mkdir /autograder ADD requirements.txt /tmp/requirements.txt RUN pip install -r /tmp/requirements.txt -RUN pip install otter-grader==2.2.1 +RUN pip install git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2 diff --git a/otter/generate/templates/python/requirements.txt b/otter/generate/templates/python/requirements.txt index e10eb4a8b..48fb2a8ec 100644 --- a/otter/generate/templates/python/requirements.txt +++ b/otter/generate/templates/python/requirements.txt @@ -13,6 +13,6 @@ nbformat dill jupytext numpy -otter-grader==2.2.1 +git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2 {% endif %}{% if other_requirements %} {{ other_requirements }}{% endif %} \ No newline at end of file diff --git a/otter/generate/templates/r/requirements.txt b/otter/generate/templates/r/requirements.txt index e4a75e5ee..1a9309550 100644 --- a/otter/generate/templates/r/requirements.txt +++ b/otter/generate/templates/r/requirements.txt @@ -14,4 +14,4 @@ dill rpy2 jupytext numpy -otter-grader==2.2.1 +git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2 diff --git a/test/test-assign/gs-autograder-correct/requirements.txt b/test/test-assign/gs-autograder-correct/requirements.txt index 67d4a5fb6..64aec99d0 100644 --- a/test/test-assign/gs-autograder-correct/requirements.txt +++ b/test/test-assign/gs-autograder-correct/requirements.txt @@ -13,4 +13,4 @@ nbformat dill jupytext numpy -otter-grader==2.2.1 +git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2 diff --git a/test/test-assign/rmd-autograder-correct/requirements.txt b/test/test-assign/rmd-autograder-correct/requirements.txt index 1e3f0a988..3018a42a8 100644 --- a/test/test-assign/rmd-autograder-correct/requirements.txt +++ b/test/test-assign/rmd-autograder-correct/requirements.txt @@ -14,4 +14,4 @@ dill rpy2 jupytext numpy -otter-grader==2.2.1 \ No newline at end of file +git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2 \ No newline at end of file diff --git a/test/test-run/autograder/source/requirements.txt b/test/test-run/autograder/source/requirements.txt index 5a58e1c2d..3f56606be 100644 --- a/test/test-run/autograder/source/requirements.txt +++ b/test/test-run/autograder/source/requirements.txt @@ -13,6 +13,6 @@ nbformat dill jupytext numpy -otter-grader==2.2.1 +git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2 tqdm diff --git a/test/test_generate/test-autograder/autograder-correct/requirements.txt b/test/test_generate/test-autograder/autograder-correct/requirements.txt index 5a58e1c2d..3f56606be 100644 --- a/test/test_generate/test-autograder/autograder-correct/requirements.txt +++ b/test/test_generate/test-autograder/autograder-correct/requirements.txt @@ -13,6 +13,6 @@ nbformat dill jupytext numpy -otter-grader==2.2.1 +git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2 tqdm diff --git a/test/test_generate/test-autograder/autograder-custom-env/requirements.txt b/test/test_generate/test-autograder/autograder-custom-env/requirements.txt index 5a58e1c2d..3f56606be 100644 --- a/test/test_generate/test-autograder/autograder-custom-env/requirements.txt +++ b/test/test_generate/test-autograder/autograder-custom-env/requirements.txt @@ -13,6 +13,6 @@ nbformat dill jupytext numpy -otter-grader==2.2.1 +git+https://github.com/ucbds-infra/otter-grader.git@54636f71f6f759bfa7593d11598b83927bc358a2 tqdm From dce92244308c7d953283205ea9d0e92682b4702f Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Tue, 15 Jun 2021 21:11:33 -0700 Subject: [PATCH 3/4] changelog --- docs/changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 317920ac3..5056b5d38 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ **Unreleased:** +* Added `show_all_public` configuration for grading + +**v2.2.1:** + * Added metadata for each test case to Otter Assign format * Added success and failure messages for test cases * Added Gmail notifications plugin From 5100702b660420137a404f921af1023bb99d1b6c Mon Sep 17 00:00:00 2001 From: Chris Pyles Date: Tue, 15 Jun 2021 22:31:51 -0700 Subject: [PATCH 4/4] fixing bug --- otter/test_files/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otter/test_files/__init__.py b/otter/test_files/__init__.py index 9e42811e7..f2d28491c 100644 --- a/otter/test_files/__init__.py +++ b/otter/test_files/__init__.py @@ -247,7 +247,7 @@ def to_gradescope_dict(self, config): hidden_test_visibility = ("hidden", "after_published")[options["show_hidden"]] # if show_all_public is true and all tests are public tests, display all tests in results - if options["show_all_public"] and all(tf.all_public for tf in self.test_files.values()): + if options["show_all_public"] and all(tf.all_public for tf in self.results.values()): hidden_test_visibility = "visible" # start w/ summary of public tests