From cfe8b03572554f66a515015aca7d4dc88142142f Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 23 Nov 2018 15:44:20 -0500 Subject: [PATCH 01/21] add option to identify the relative of the fifth --- madmom/evaluation/key.py | 115 +++++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 29 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index b728b7fcb..edbaac4fe 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -59,19 +59,23 @@ def key_label_to_class(key_label): return key_class -def error_type(det_key, ann_key, strict_fifth=False): +def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): """ - Compute the evaluation score and error category for a predicted key - compared to the annotated key. + Compute the error category for a predicted key compared to the annotated key. - Categories and evaluation scores follow the evaluation strategy used + Categories follow the evaluation strategy used for MIREX (see http://music-ir.org/mirex/wiki/2017:Audio_Key_Detection). + There are two evaluation modes for the 'fifth' category: by default, a detection falls into the 'fifth' category if it is the fifth of the annotation, or the annotation is the fifth of the detection. If `strict_fifth` is `True`, only the former case is considered. This is the mode used for MIREX. + There is an optional category: relative of fifth. This allows to separate + keys that are closely related to the annotated key on the circle of fifth + from the 'other' error category. + Parameters ---------- det_key : int @@ -80,12 +84,46 @@ def error_type(det_key, ann_key, strict_fifth=False): Annotated key class. strict_fifth: bool Use strict interpretation of the 'fifth' category, as in MIREX. + relative_of_fifth: bool + Differentiate relative keys of the fifth wrt the annotated key. Returns ------- - score, category : float, str - Evaluation score and error category. + category : str + Error category. + + Examples + -------- + >>> from madmom.evaluation.key import error_type + + # annotated: 'C major' / detected: 'C major' + >>> error_type(0, 0) + 'correct' + + # annotated: 'C major' / detected: 'G major': + 7 semitones from annotated key + >>> error_type(7,0) + 'fifth' + # annotated: 'C major' / detected: 'F major': -7 semitones from annotated key (modulo 12) + >>> error_type(5,0) + 'fifth' + + # annotated: 'C major' / detected: 'F major': -7 semitones from annotated key (modulo 12), the MIREX way + >>> error_type(5, 0, strict_fifth=True) + 'other' + + # annotated: 'C major' / detected: 'E minor': E minor is the relative key of G Major, which is the fifth of C Major + >>> error_type(16, 0, relative_of_fifth=True) + 'relative_of_fifth' + + # annotated: 'C major' / detected: 'D minor': D minor is the relative key of F Major, of which C Major is the fifth + >>> error_type(14, 0, relative_of_fifth=True) + 'relative_of_fifth' + + # annotated: 'C major' / detected: 'D minor': D minor is the relative key of F Major, of which C Major is the fifth + # - using MIREX definition of 'fifth' + >>> error_type(14, 0, relative_of_fifth=True, strict_fifth=True) + 'other' """ ann_root = ann_key % 12 ann_mode = ann_key // 12 @@ -94,23 +132,34 @@ def error_type(det_key, ann_key, strict_fifth=False): major, minor = 0, 1 if det_root == ann_root and det_mode == ann_mode: - return 1.0, 'correct' - if det_mode == ann_mode and ((det_root - ann_root) % 12 == 7): - return 0.5, 'fifth' - if not strict_fifth and (det_mode == ann_mode and - ((det_root - ann_root) % 12 == 5)): - return 0.5, 'fifth' - if (ann_mode == major and det_mode != ann_mode and ( - (det_root - ann_root) % 12 == 9)): - return 0.3, 'relative' - if (ann_mode == minor and det_mode != ann_mode and ( - (det_root - ann_root) % 12 == 3)): - return 0.3, 'relative' - if det_mode != ann_mode and det_root == ann_root: - return 0.2, 'parallel' + return 'correct' + elif det_mode == ann_mode and ((det_root - ann_root) % 12 == 7): + return 'fifth' + elif not strict_fifth and \ + (det_mode == ann_mode and ((det_root - ann_root) % 12 == 5)): + return 'fifth' + elif (ann_mode == major and det_mode != ann_mode and ((det_root - ann_root) % 12 == 9)): + return 'relative' + elif (ann_mode == minor and det_mode != ann_mode and ((det_root - ann_root) % 12 == 3)): + return 'relative' + elif relative_of_fifth and \ + (ann_mode == major and det_mode != ann_mode and ((det_root - ann_root) % 12 == 4)): + return 'relative_of_fifth' + elif not strict_fifth and \ + (relative_of_fifth and (ann_mode == major and det_mode != ann_mode and ( + (det_root - ann_root) % 12 == 2))): + return 'relative_of_fifth' + elif relative_of_fifth and (ann_mode == minor and det_mode != ann_mode and ( + (det_root - ann_root) % 12 == 10)): + return 'relative_of_fifth' + elif not strict_fifth and \ + (relative_of_fifth and (ann_mode == minor and det_mode != ann_mode and ( + (det_root - ann_root) % 12 == 8))): + return 'relative_of_fifth' + elif det_mode != ann_mode and det_root == ann_root: + return 'parallel' else: - return 0.0, 'other' - + return 'other' class KeyEvaluation(EvaluationMixin): """ @@ -134,14 +183,19 @@ class KeyEvaluation(EvaluationMixin): ('error_category', 'Error Category') ] - def __init__(self, detection, annotation, strict_fifth=False, name=None, - **kwargs): + error_scores = {'correct': 1.0, + 'fifth': 0.5, + 'relative': 0.3, + 'relative_of_fifth': 0.0, + 'parallel': 0.2, + 'other': 0.0} + + def __init__(self, detection, annotation, strict_fifth=False, name=None, relative_of_fifth=False, **kwargs): self.name = name or '' self.detection = key_label_to_class(detection) self.annotation = key_label_to_class(annotation) - self.score, self.error_category = error_type( - self.detection, self.annotation, strict_fifth - ) + self.error_category = error_type(self.detection, self.annotation, strict_fifth, relative_of_fifth) + self.score = self.error_scores.get(self.error_category, 0.0) def tostring(self, **kwargs): """ @@ -175,6 +229,7 @@ class KeyMeanEvaluation(EvaluationMixin): ('correct', 'Correct'), ('fifth', 'Fifth'), ('relative', 'Relative'), + ('relative_of_fifth', 'Relative of Fifth'), ('parallel', 'Parallel'), ('other', 'Other'), ('weighted', 'Weighted'), @@ -189,15 +244,17 @@ def __init__(self, eval_objects, name=None): self.correct = float(c['correct']) / n self.fifth = float(c['fifth']) / n self.relative = float(c['relative']) / n + self.relative_of_fifth= float(c['relative_of_fifth']) / n self.parallel = float(c['parallel']) / n self.other = float(c['other']) / n self.weighted = sum(e.score for e in eval_objects) / n def tostring(self, **kwargs): return ('{}\n Weighted: {:.3f} Correct: {:.3f} Fifth: {:.3f} ' - 'Relative: {:.3f} Parallel: {:.3f} Other: {:.3f}'.format( + 'Relative: {:.3f} Relative of Fifth: {:.3f} ' + 'Parallel: {:.3f} Other: {:.3f}'.format( self.name, self.weighted, self.correct, self.fifth, - self.relative, self.parallel, self.other)) + self.relative, self.relative_of_fifth, self.parallel, self.other)) def add_parser(parser): From 561c80a3879e705dc8cd3f738982c967c0169e44 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 23 Nov 2018 15:45:30 -0500 Subject: [PATCH 02/21] update tests: now the KeyEvalutation test has to check the scores for each error category so new files have been added to the detection folder. --- tests/data/detections/dummy.key.correct.txt | 1 + tests/data/detections/dummy.key.fifth.txt | 1 + tests/data/detections/dummy.key.other.txt | 1 + tests/data/detections/dummy.key.parallel.txt | 1 + .../{dummy.key.txt => dummy.key.relative.txt} | 0 .../dummy.key.relative_of_fifth.txt | 1 + tests/test_evaluation_key.py | 163 +++++++++++++----- tests/test_io.py | 4 +- tests/test_utils.py | 2 +- 9 files changed, 131 insertions(+), 43 deletions(-) create mode 100644 tests/data/detections/dummy.key.correct.txt create mode 100644 tests/data/detections/dummy.key.fifth.txt create mode 100644 tests/data/detections/dummy.key.other.txt create mode 100644 tests/data/detections/dummy.key.parallel.txt rename tests/data/detections/{dummy.key.txt => dummy.key.relative.txt} (100%) create mode 100644 tests/data/detections/dummy.key.relative_of_fifth.txt diff --git a/tests/data/detections/dummy.key.correct.txt b/tests/data/detections/dummy.key.correct.txt new file mode 100644 index 000000000..700bb93bf --- /dev/null +++ b/tests/data/detections/dummy.key.correct.txt @@ -0,0 +1 @@ +f# minor \ No newline at end of file diff --git a/tests/data/detections/dummy.key.fifth.txt b/tests/data/detections/dummy.key.fifth.txt new file mode 100644 index 000000000..07413433e --- /dev/null +++ b/tests/data/detections/dummy.key.fifth.txt @@ -0,0 +1 @@ +B minor \ No newline at end of file diff --git a/tests/data/detections/dummy.key.other.txt b/tests/data/detections/dummy.key.other.txt new file mode 100644 index 000000000..45f4b6947 --- /dev/null +++ b/tests/data/detections/dummy.key.other.txt @@ -0,0 +1 @@ +G# minor \ No newline at end of file diff --git a/tests/data/detections/dummy.key.parallel.txt b/tests/data/detections/dummy.key.parallel.txt new file mode 100644 index 000000000..d11999193 --- /dev/null +++ b/tests/data/detections/dummy.key.parallel.txt @@ -0,0 +1 @@ +F# maj \ No newline at end of file diff --git a/tests/data/detections/dummy.key.txt b/tests/data/detections/dummy.key.relative.txt similarity index 100% rename from tests/data/detections/dummy.key.txt rename to tests/data/detections/dummy.key.relative.txt diff --git a/tests/data/detections/dummy.key.relative_of_fifth.txt b/tests/data/detections/dummy.key.relative_of_fifth.txt new file mode 100644 index 000000000..db05a7ec8 --- /dev/null +++ b/tests/data/detections/dummy.key.relative_of_fifth.txt @@ -0,0 +1 @@ +e maj \ No newline at end of file diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index f0dc0fb14..368580ba9 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -62,35 +62,47 @@ def test_values(self): class TestErrorTypeFunction(unittest.TestCase): def _compare_scores(self, correct, fifth_strict, fifth_lax, relative, - parallel): + relative_of_fifth_up, relative_of_fifth_down, parallel): for det_key in range(24): - score, cat = error_type(det_key, correct) - score_st, cat_st = error_type(det_key, correct, strict_fifth=True) + cat = error_type(det_key, correct) + cat_st = error_type(det_key, correct, strict_fifth=True) + cat_rf = error_type(det_key, correct, relative_of_fifth=True) + cat_st_rf = error_type(det_key, correct, strict_fifth=True, relative_of_fifth=True) if det_key == correct: self.assertEqual(cat, 'correct') - self.assertEqual(score, 1.0) self.assertEqual(cat_st, cat) - self.assertEqual(score_st, score) - if det_key == fifth_strict: + self.assertEqual(cat_rf, cat) + self.assertEqual(cat_st_rf, cat) + elif det_key == fifth_strict: self.assertEqual(cat, 'fifth') - self.assertEqual(score, 0.5) self.assertEqual(cat_st, cat) - self.assertEqual(score_st, score) - if det_key == fifth_lax: + self.assertEqual(cat_rf, cat) + self.assertEqual(cat_st_rf, cat) + elif det_key == fifth_lax: self.assertEqual(cat, 'fifth') - self.assertEqual(score, 0.5) self.assertEqual(cat_st, 'other') - self.assertEqual(score_st, 0.0) - if det_key == relative: + self.assertEqual(cat_rf, 'fifth') + self.assertEqual(cat_st_rf, 'other') + elif det_key == relative: self.assertEqual(cat, 'relative') - self.assertEqual(score, 0.3) self.assertEqual(cat_st, cat) - self.assertEqual(score_st, score) - if det_key == parallel: + self.assertEqual(cat_rf, cat) + self.assertEqual(cat_st_rf, cat) + elif det_key == relative_of_fifth_down: + self.assertEqual(cat, 'other') + self.assertEqual(cat_st, cat) + self.assertEqual(cat_rf, 'relative_of_fifth') + self.assertEqual(cat_st_rf, cat) + elif det_key == relative_of_fifth_up: + self.assertEqual(cat, 'other') + self.assertEqual(cat_st, cat) + self.assertEqual(cat_rf, 'relative_of_fifth') + self.assertEqual(cat_st_rf, 'relative_of_fifth') + elif det_key == parallel: self.assertEqual(cat, 'parallel') - self.assertEqual(score, 0.2) self.assertEqual(cat_st, cat) - self.assertEqual(score_st, score) + self.assertEqual(cat_rf, cat) + self.assertEqual(cat_st_rf, cat) def test_values(self): self._compare_scores( @@ -98,6 +110,8 @@ def test_values(self): fifth_strict=key_label_to_class('g maj'), fifth_lax=key_label_to_class('f maj'), relative=key_label_to_class('a min'), + relative_of_fifth_up=key_label_to_class('e min'), + relative_of_fifth_down=key_label_to_class('d min'), parallel=key_label_to_class('c min') ) @@ -106,6 +120,8 @@ def test_values(self): fifth_strict=key_label_to_class('bb maj'), fifth_lax=key_label_to_class('ab maj'), relative=key_label_to_class('c min'), + relative_of_fifth_up=key_label_to_class('g min'), + relative_of_fifth_down=key_label_to_class('f min'), parallel=key_label_to_class('eb min') ) @@ -114,6 +130,8 @@ def test_values(self): fifth_strict=key_label_to_class('e min'), fifth_lax=key_label_to_class('d min'), relative=key_label_to_class('c maj'), + relative_of_fifth_up=key_label_to_class('g maj'), + relative_of_fifth_down=key_label_to_class('f maj'), parallel=key_label_to_class('a maj') ) @@ -122,6 +140,8 @@ def test_values(self): fifth_strict=key_label_to_class('gb min'), fifth_lax=key_label_to_class('e min'), relative=key_label_to_class('d maj'), + relative_of_fifth_up=key_label_to_class('a maj'), + relative_of_fifth_down=key_label_to_class('g maj'), parallel=key_label_to_class('b maj') ) @@ -129,46 +149,109 @@ def test_values(self): class TestKeyEvaluationClass(unittest.TestCase): def setUp(self): - self.eval = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.txt')), + # this one should have a score of 1 + self.eval_correct = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.correct.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), - name='TestEval' + name='eval_correct' + ) + # this one should have a score of 0.5 + self.eval_fifth = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.fifth.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + name='eval_fifth' + ) + + # this one should have a score of 0.3 + self.eval_relative = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.relative.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + name='eval_relative' + ) + # this one should have a score of 0.2 + self.eval_parallel = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.parallel.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + name='eval_parallel' + ) + # this one should have a score of 0.0 + self.eval_relative_of_fifth = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.relative_of_fifth.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + relative_of_fifth=True, + name='eval_relative_of_fifth' + ) + # this one should have a score of 0.0 + self.eval_other = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.other.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + name='eval_other' ) def test_init(self): - self.assertTrue(self.eval.name == 'TestEval') - self.assertTrue(self.eval.detection, 9) - self.assertTrue(self.eval.annotation, 18) + self.assertTrue(self.eval_relative.name == 'eval_relative') + self.assertTrue(self.eval_relative.detection, 9) + self.assertTrue(self.eval_relative.annotation, 18) def test_results(self): - self.assertEqual(self.eval.error_category, 'relative') - self.assertEqual(self.eval.score, 0.3) + # Correct + self.assertEqual(self.eval_correct.error_category, 'correct') + self.assertEqual(self.eval_correct.score, 1.0) + # Fifth + self.assertEqual(self.eval_fifth.error_category, 'fifth') + self.assertEqual(self.eval_fifth.score, 0.5) + # Relative + self.assertEqual(self.eval_relative.error_category, 'relative') + self.assertEqual(self.eval_relative.score, 0.3) + # Relative of Fifth + self.assertEqual(self.eval_relative_of_fifth.error_category, 'relative_of_fifth') + self.assertEqual(self.eval_relative_of_fifth.score, 0.0) + # Parallel + self.assertEqual(self.eval_parallel.error_category, 'parallel') + self.assertEqual(self.eval_parallel.score, 0.2) + # Other + self.assertEqual(self.eval_other.error_category, 'other') + self.assertEqual(self.eval_other.score, 0.0) class TestKeyMeanEvaluation(unittest.TestCase): def setUp(self): # this one should have a score of 1 - self.eval1 = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.txt')), - load_key(join(DETECTIONS_PATH, 'dummy.key.txt')), - name='eval1' + self.eval_correct = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.correct.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + name='eval_correct' ) - # this one should have a score of 0.3 - self.eval2 = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.txt')), + # this one should have a score of 0.2 + self.eval_parallel = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.parallel.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + name='eval_parallel' + ) + # this one should have a score of 0.0 + self.eval_relative = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.relative.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + name='eval_relative' + ) + # this one should have a score of 0.0 + self.eval_other = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.key.other.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), - name='eval2' + name='eval_other' ) def test_mean_results(self): - mean_eval = KeyMeanEvaluation([self.eval1, self.eval2]) - self.assertAlmostEqual(mean_eval.correct, 0.5) - self.assertAlmostEqual(mean_eval.fifth, 0.) - self.assertAlmostEqual(mean_eval.relative, 0.5) - self.assertAlmostEqual(mean_eval.parallel, 0.0) - self.assertAlmostEqual(mean_eval.other, 0.0) - self.assertAlmostEqual(mean_eval.weighted, 0.65) + evals = [self.eval_correct, self.eval_parallel, self.eval_relative, self.eval_other] + mean_eval = KeyMeanEvaluation(evals) + self.assertAlmostEqual(mean_eval.correct, 1.0 / len(evals)) + self.assertAlmostEqual(mean_eval.fifth, 0.0) + self.assertAlmostEqual(mean_eval.relative, 1.0 / len(evals)) + self.assertAlmostEqual(mean_eval.relative_of_fifth, 0.0) + self.assertAlmostEqual(mean_eval.parallel, 1.0 / len(evals)) + self.assertAlmostEqual(mean_eval.other, 1.0 / len(evals)) + self.assertAlmostEqual(mean_eval.weighted, 0.375) class TestAddParserFunction(unittest.TestCase): diff --git a/tests/test_io.py b/tests/test_io.py index 2bea9c4b7..c78f4787d 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -97,11 +97,11 @@ class TestLoadKeyFunction(unittest.TestCase): def test_load_key_from_file(self): key = load_key(join(ANNOTATIONS_PATH, 'dummy.key')) self.assertEqual(key, 'F# minor') - key = load_key(join(DETECTIONS_PATH, 'dummy.key.txt')) + key = load_key(join(DETECTIONS_PATH, 'dummy.key.relative.txt')) self.assertEqual(key, 'a maj') key = load_key(open(join(ANNOTATIONS_PATH, 'dummy.key'))) self.assertEqual(key, 'F# minor') - key = load_key(open(join(DETECTIONS_PATH, 'dummy.key.txt'))) + key = load_key(open(join(DETECTIONS_PATH, 'dummy.key.relative.txt'))) self.assertEqual(key, 'a maj') diff --git a/tests/test_utils.py b/tests/test_utils.py index 3f5bba98f..e788e46ce 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -64,7 +64,7 @@ pj(ANNOTATIONS_PATH, 'piano_sample.notes_in_beats')] DETECTION_FILES = [pj(DETECTIONS_PATH, 'dummy.chords.txt'), - pj(DETECTIONS_PATH, 'dummy.key.txt'), + pj(DETECTIONS_PATH, 'dummy.key.relative.txt'), pj(DETECTIONS_PATH, 'sample.beat_detector.txt'), pj(DETECTIONS_PATH, 'sample.beat_tracker.txt'), pj(DETECTIONS_PATH, 'sample.cnn_chord_recognition.txt'), From 849eb19bbc6744375f650f9ee434ce52d84809a3 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 23 Nov 2018 16:27:53 -0500 Subject: [PATCH 03/21] fix naming of key detection file so that it does not break other tests --- ....key.correct.txt => dummy.correct.key.txt} | 0 ...ummy.key.fifth.txt => dummy.fifth.key.txt} | 0 .../{dummy.key.relative.txt => dummy.key.txt} | 0 ...ummy.key.other.txt => dummy.other.key.txt} | 0 ...ey.parallel.txt => dummy.parallel.key.txt} | 0 ...th.txt => dummy.relative_of_fifth.key.txt} | 0 tests/test_bin_evaluate.py | 2 +- tests/test_evaluation_key.py | 20 +++++++++---------- tests/test_io.py | 4 ++-- tests/test_utils.py | 7 ++++++- 10 files changed, 19 insertions(+), 14 deletions(-) rename tests/data/detections/{dummy.key.correct.txt => dummy.correct.key.txt} (100%) rename tests/data/detections/{dummy.key.fifth.txt => dummy.fifth.key.txt} (100%) rename tests/data/detections/{dummy.key.relative.txt => dummy.key.txt} (100%) rename tests/data/detections/{dummy.key.other.txt => dummy.other.key.txt} (100%) rename tests/data/detections/{dummy.key.parallel.txt => dummy.parallel.key.txt} (100%) rename tests/data/detections/{dummy.key.relative_of_fifth.txt => dummy.relative_of_fifth.key.txt} (100%) diff --git a/tests/data/detections/dummy.key.correct.txt b/tests/data/detections/dummy.correct.key.txt similarity index 100% rename from tests/data/detections/dummy.key.correct.txt rename to tests/data/detections/dummy.correct.key.txt diff --git a/tests/data/detections/dummy.key.fifth.txt b/tests/data/detections/dummy.fifth.key.txt similarity index 100% rename from tests/data/detections/dummy.key.fifth.txt rename to tests/data/detections/dummy.fifth.key.txt diff --git a/tests/data/detections/dummy.key.relative.txt b/tests/data/detections/dummy.key.txt similarity index 100% rename from tests/data/detections/dummy.key.relative.txt rename to tests/data/detections/dummy.key.txt diff --git a/tests/data/detections/dummy.key.other.txt b/tests/data/detections/dummy.other.key.txt similarity index 100% rename from tests/data/detections/dummy.key.other.txt rename to tests/data/detections/dummy.other.key.txt diff --git a/tests/data/detections/dummy.key.parallel.txt b/tests/data/detections/dummy.parallel.key.txt similarity index 100% rename from tests/data/detections/dummy.key.parallel.txt rename to tests/data/detections/dummy.parallel.key.txt diff --git a/tests/data/detections/dummy.key.relative_of_fifth.txt b/tests/data/detections/dummy.relative_of_fifth.key.txt similarity index 100% rename from tests/data/detections/dummy.key.relative_of_fifth.txt rename to tests/data/detections/dummy.relative_of_fifth.key.txt diff --git a/tests/test_bin_evaluate.py b/tests/test_bin_evaluate.py index 7bfa5f367..701e90a08 100644 --- a/tests/test_bin_evaluate.py +++ b/tests/test_bin_evaluate.py @@ -87,7 +87,7 @@ def test_key(self): res = run_script('key') # second line contains the results res = np.fromiter(res[1].split(',')[1:], dtype=np.float) - self.assertTrue(np.allclose(res, [0, 0, 1, 0, 0, 0.3])) + self.assertTrue(np.allclose(res, [0, 0, 1, 0, 0, 0, 0.3])) def test_notes(self): res = run_script('notes', det_suffix='.piano_transcriptor.txt') diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index 368580ba9..8af8ed7c9 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -151,39 +151,39 @@ class TestKeyEvaluationClass(unittest.TestCase): def setUp(self): # this one should have a score of 1 self.eval_correct = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.correct.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.correct.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_correct' ) # this one should have a score of 0.5 self.eval_fifth = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.fifth.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.fifth.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_fifth' ) # this one should have a score of 0.3 self.eval_relative = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.relative.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_relative' ) # this one should have a score of 0.2 self.eval_parallel = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.parallel.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.parallel.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_parallel' ) # this one should have a score of 0.0 self.eval_relative_of_fifth = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.relative_of_fifth.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.relative_of_fifth.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), relative_of_fifth=True, name='eval_relative_of_fifth' ) # this one should have a score of 0.0 self.eval_other = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.other.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.other.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_other' ) @@ -219,25 +219,25 @@ class TestKeyMeanEvaluation(unittest.TestCase): def setUp(self): # this one should have a score of 1 self.eval_correct = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.correct.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.correct.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_correct' ) # this one should have a score of 0.2 self.eval_parallel = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.parallel.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.parallel.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_parallel' ) # this one should have a score of 0.0 self.eval_relative = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.relative.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_relative' ) # this one should have a score of 0.0 self.eval_other = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.key.other.txt')), + load_key(join(DETECTIONS_PATH, 'dummy.other.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_other' ) diff --git a/tests/test_io.py b/tests/test_io.py index c78f4787d..2bea9c4b7 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -97,11 +97,11 @@ class TestLoadKeyFunction(unittest.TestCase): def test_load_key_from_file(self): key = load_key(join(ANNOTATIONS_PATH, 'dummy.key')) self.assertEqual(key, 'F# minor') - key = load_key(join(DETECTIONS_PATH, 'dummy.key.relative.txt')) + key = load_key(join(DETECTIONS_PATH, 'dummy.key.txt')) self.assertEqual(key, 'a maj') key = load_key(open(join(ANNOTATIONS_PATH, 'dummy.key'))) self.assertEqual(key, 'F# minor') - key = load_key(open(join(DETECTIONS_PATH, 'dummy.key.relative.txt'))) + key = load_key(open(join(DETECTIONS_PATH, 'dummy.key.txt'))) self.assertEqual(key, 'a maj') diff --git a/tests/test_utils.py b/tests/test_utils.py index e788e46ce..0847d9c3a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -64,7 +64,12 @@ pj(ANNOTATIONS_PATH, 'piano_sample.notes_in_beats')] DETECTION_FILES = [pj(DETECTIONS_PATH, 'dummy.chords.txt'), - pj(DETECTIONS_PATH, 'dummy.key.relative.txt'), + pj(DETECTIONS_PATH, 'dummy.key.txt'), + pj(DETECTIONS_PATH, 'dummy.correct.key.txt'), + pj(DETECTIONS_PATH, 'dummy.fifth.key.txt'), + pj(DETECTIONS_PATH, 'dummy.other.key.txt'), + pj(DETECTIONS_PATH, 'dummy.parallel.key.txt'), + pj(DETECTIONS_PATH, 'dummy.relative_of_fifth.key.txt'), pj(DETECTIONS_PATH, 'sample.beat_detector.txt'), pj(DETECTIONS_PATH, 'sample.beat_tracker.txt'), pj(DETECTIONS_PATH, 'sample.cnn_chord_recognition.txt'), From 30123e7a8ee152b04f3a55e2a87282b8ae51e8b3 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Mon, 26 Nov 2018 13:00:47 -0500 Subject: [PATCH 04/21] add a check that each KeyEvaluation in a list passed to the KeyMeanEvaluation class uses exactly the same error_scores --- madmom/evaluation/key.py | 25 ++++++++++++++++++++++--- tests/test_evaluation_key.py | 12 ++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index edbaac4fe..a92444d46 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -72,7 +72,7 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): If `strict_fifth` is `True`, only the former case is considered. This is the mode used for MIREX. - There is an optional category: relative of fifth. This allows to separate + There is an optional category: 'relative of fifth'. This allows to separate keys that are closely related to the annotated key on the circle of fifth from the 'other' error category. @@ -85,7 +85,8 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): strict_fifth: bool Use strict interpretation of the 'fifth' category, as in MIREX. relative_of_fifth: bool - Differentiate relative keys of the fifth wrt the annotated key. + Differentiate relative keys of the fifth wrt the annotated key. Is coherent with strict_fifth in the sense that + it only considers the relative key of the strict fifth. Returns ------- @@ -247,7 +248,25 @@ def __init__(self, eval_objects, name=None): self.relative_of_fifth= float(c['relative_of_fifth']) / n self.parallel = float(c['parallel']) / n self.other = float(c['other']) / n - self.weighted = sum(e.score for e in eval_objects) / n + # Check that all the error_scores in the eval_objects are the same + # (otherwise a weighted result would be hard to interpret) + if self._check_error_scores(eval_objects): + self.weighted = sum(e.score for e in eval_objects) / n + else: + raise ValueError('Different error_scores found in the KeyEvaluation objects.') + + + def _check_error_scores(self, eval_objects): + all_the_same = True + indScores = 0 + while (indScores < len(eval_objects)-1) and all_the_same: + if eval_objects[indScores].error_scores != eval_objects[indScores+1].error_scores: + all_the_same = False + break + else: + indScores+=1 + return all_the_same + def tostring(self, **kwargs): return ('{}\n Weighted: {:.3f} Correct: {:.3f} Fifth: {:.3f} ' diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index 8af8ed7c9..9737c54d8 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -241,6 +241,18 @@ def setUp(self): load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_other' ) + # this one has has the same key BUT a different set of error scores + self.eval_different_scores = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.correct.key.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + name='eval_correct_different_scores' + ) + self.eval_different_scores.error_scores={'correct':0.5} + + def test_check_error_scores(self): + evals = [self.eval_correct, self.eval_parallel, self.eval_different_scores, self.eval_other] + with self.assertRaises(ValueError): + KeyMeanEvaluation(evals) def test_mean_results(self): evals = [self.eval_correct, self.eval_parallel, self.eval_relative, self.eval_other] From 50b357197f9d9b4cf97f1eb8489f2b0634b140c4 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Thu, 29 Nov 2018 21:27:35 -0500 Subject: [PATCH 05/21] KeyMeanEvaluation can now handle empty evaluation lists. --- madmom/evaluation/key.py | 29 ++++++++++++++++------------- tests/test_evaluation_key.py | 4 ++++ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index a92444d46..f9a057f23 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -240,20 +240,23 @@ def __init__(self, eval_objects, name=None): self.name = name or 'mean for {:d} files'.format(len(eval_objects)) n = len(eval_objects) - c = Counter(e.error_category for e in eval_objects) - - self.correct = float(c['correct']) / n - self.fifth = float(c['fifth']) / n - self.relative = float(c['relative']) / n - self.relative_of_fifth= float(c['relative_of_fifth']) / n - self.parallel = float(c['parallel']) / n - self.other = float(c['other']) / n - # Check that all the error_scores in the eval_objects are the same - # (otherwise a weighted result would be hard to interpret) - if self._check_error_scores(eval_objects): - self.weighted = sum(e.score for e in eval_objects) / n + if n > 0: + c = Counter(e.error_category for e in eval_objects) + + self.correct = float(c['correct']) / n + self.fifth = float(c['fifth']) / n + self.relative = float(c['relative']) / n + self.relative_of_fifth= float(c['relative_of_fifth']) / n + self.parallel = float(c['parallel']) / n + self.other = float(c['other']) / n + # Check that all the error_scores in the eval_objects are the same + # (otherwise a weighted result would be hard to interpret) + if self._check_error_scores(eval_objects): + self.weighted = sum(e.score for e in eval_objects) / n + else: + raise ValueError('Different error_scores found in the KeyEvaluation objects.') else: - raise ValueError('Different error_scores found in the KeyEvaluation objects.') + raise ValueError('The list of evaluations is empty.') def _check_error_scores(self, eval_objects): diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index 9737c54d8..7859d6a91 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -254,6 +254,10 @@ def test_check_error_scores(self): with self.assertRaises(ValueError): KeyMeanEvaluation(evals) + def test_empty_eval_list(self): + with self.assertRaises(ValueError): + KeyMeanEvaluation([]) + def test_mean_results(self): evals = [self.eval_correct, self.eval_parallel, self.eval_relative, self.eval_other] mean_eval = KeyMeanEvaluation(evals) From 2fab590aca120cf83e9d79696a5e148ea7fea770 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 30 Nov 2018 17:04:24 -0500 Subject: [PATCH 06/21] fixing code style issues + "too many returns" --- madmom/evaluation/key.py | 107 ++++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index f9a057f23..001cb7057 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -61,7 +61,8 @@ def key_label_to_class(key_label): def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): """ - Compute the error category for a predicted key compared to the annotated key. + Compute the error category for a predicted key compared to + the annotated key. Categories follow the evaluation strategy used for MIREX (see http://music-ir.org/mirex/wiki/2017:Audio_Key_Detection). @@ -85,8 +86,9 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): strict_fifth: bool Use strict interpretation of the 'fifth' category, as in MIREX. relative_of_fifth: bool - Differentiate relative keys of the fifth wrt the annotated key. Is coherent with strict_fifth in the sense that - it only considers the relative key of the strict fifth. + Differentiate relative keys of the fifth wrt the annotated key. + Is coherent with strict_fifth in the sense that it only considers the + relative key of the strict fifth. Returns ------- @@ -101,27 +103,31 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): >>> error_type(0, 0) 'correct' - # annotated: 'C major' / detected: 'G major': + 7 semitones from annotated key + # annotated: 'C major' / detected: 'G major': +7 semitones >>> error_type(7,0) 'fifth' - # annotated: 'C major' / detected: 'F major': -7 semitones from annotated key (modulo 12) + # annotated: 'C major' / detected: 'F major': -7 semitones (modulo 12) >>> error_type(5,0) 'fifth' - # annotated: 'C major' / detected: 'F major': -7 semitones from annotated key (modulo 12), the MIREX way + # annotated: 'C major' / detected: 'F major': -7 semitones (modulo 12), + # the MIREX way >>> error_type(5, 0, strict_fifth=True) 'other' - # annotated: 'C major' / detected: 'E minor': E minor is the relative key of G Major, which is the fifth of C Major + # annotated: 'C major' / detected: 'E minor': E minor is the relative key + # of G Major, which is the fifth of C Major >>> error_type(16, 0, relative_of_fifth=True) 'relative_of_fifth' - # annotated: 'C major' / detected: 'D minor': D minor is the relative key of F Major, of which C Major is the fifth + # annotated: 'C major' / detected: 'D minor': D minor is the relative key + # of F Major, of which C Major is the fifth >>> error_type(14, 0, relative_of_fifth=True) 'relative_of_fifth' - # annotated: 'C major' / detected: 'D minor': D minor is the relative key of F Major, of which C Major is the fifth + # annotated: 'C major' / detected: 'D minor': D minor is the relative key + # of F Major, of which C Major is the fifth # - using MIREX definition of 'fifth' >>> error_type(14, 0, relative_of_fifth=True, strict_fifth=True) 'other' @@ -133,34 +139,43 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): major, minor = 0, 1 if det_root == ann_root and det_mode == ann_mode: - return 'correct' + error_type = 'correct' elif det_mode == ann_mode and ((det_root - ann_root) % 12 == 7): - return 'fifth' + error_type = 'fifth' elif not strict_fifth and \ (det_mode == ann_mode and ((det_root - ann_root) % 12 == 5)): - return 'fifth' - elif (ann_mode == major and det_mode != ann_mode and ((det_root - ann_root) % 12 == 9)): - return 'relative' - elif (ann_mode == minor and det_mode != ann_mode and ((det_root - ann_root) % 12 == 3)): - return 'relative' - elif relative_of_fifth and \ - (ann_mode == major and det_mode != ann_mode and ((det_root - ann_root) % 12 == 4)): - return 'relative_of_fifth' + error_type = 'fifth' + elif (ann_mode == major and det_mode != ann_mode + and ((det_root - ann_root) % 12 == 9)): + error_type = 'relative' + elif (ann_mode == minor and det_mode != ann_mode + and ((det_root - ann_root) % 12 == 3)): + error_type = 'relative' + elif relative_of_fifth \ + and (ann_mode == major and det_mode != ann_mode + and ((det_root - ann_root) % 12 == 4)): + error_type = 'relative_of_fifth' elif not strict_fifth and \ - (relative_of_fifth and (ann_mode == major and det_mode != ann_mode and ( - (det_root - ann_root) % 12 == 2))): - return 'relative_of_fifth' - elif relative_of_fifth and (ann_mode == minor and det_mode != ann_mode and ( - (det_root - ann_root) % 12 == 10)): - return 'relative_of_fifth' + (relative_of_fifth and (ann_mode == major + and det_mode != ann_mode + and ((det_root - ann_root) % 12 == 2))): + error_type = 'relative_of_fifth' + elif relative_of_fifth \ + and (ann_mode == minor and det_mode != ann_mode + and ((det_root - ann_root) % 12 == 10)): + error_type = 'relative_of_fifth' elif not strict_fifth and \ - (relative_of_fifth and (ann_mode == minor and det_mode != ann_mode and ( - (det_root - ann_root) % 12 == 8))): - return 'relative_of_fifth' + (relative_of_fifth and (ann_mode == minor + and det_mode != ann_mode + and ((det_root - ann_root) % 12 == 8))): + error_type = 'relative_of_fifth' elif det_mode != ann_mode and det_root == ann_root: - return 'parallel' + error_type = 'parallel' else: - return 'other' + error_type = 'other' + + return error_type + class KeyEvaluation(EvaluationMixin): """ @@ -191,11 +206,19 @@ class KeyEvaluation(EvaluationMixin): 'parallel': 0.2, 'other': 0.0} - def __init__(self, detection, annotation, strict_fifth=False, name=None, relative_of_fifth=False, **kwargs): + def __init__(self, detection, + annotation, + strict_fifth=False, + name=None, + relative_of_fifth=False, + **kwargs): self.name = name or '' self.detection = key_label_to_class(detection) self.annotation = key_label_to_class(annotation) - self.error_category = error_type(self.detection, self.annotation, strict_fifth, relative_of_fifth) + self.error_category = error_type(self.detection, + self.annotation, + strict_fifth, + relative_of_fifth) self.score = self.error_scores.get(self.error_category, 0.0) def tostring(self, **kwargs): @@ -246,7 +269,7 @@ def __init__(self, eval_objects, name=None): self.correct = float(c['correct']) / n self.fifth = float(c['fifth']) / n self.relative = float(c['relative']) / n - self.relative_of_fifth= float(c['relative_of_fifth']) / n + self.relative_of_fifth = float(c['relative_of_fifth']) / n self.parallel = float(c['parallel']) / n self.other = float(c['other']) / n # Check that all the error_scores in the eval_objects are the same @@ -254,29 +277,31 @@ def __init__(self, eval_objects, name=None): if self._check_error_scores(eval_objects): self.weighted = sum(e.score for e in eval_objects) / n else: - raise ValueError('Different error_scores found in the KeyEvaluation objects.') + raise ValueError('Different error_scores found in the ' + 'KeyEvaluation objects.') else: raise ValueError('The list of evaluations is empty.') - def _check_error_scores(self, eval_objects): all_the_same = True indScores = 0 while (indScores < len(eval_objects)-1) and all_the_same: - if eval_objects[indScores].error_scores != eval_objects[indScores+1].error_scores: + if eval_objects[indScores].error_scores \ + != eval_objects[indScores+1].error_scores: all_the_same = False break else: - indScores+=1 + indScores += 1 return all_the_same - def tostring(self, **kwargs): return ('{}\n Weighted: {:.3f} Correct: {:.3f} Fifth: {:.3f} ' 'Relative: {:.3f} Relative of Fifth: {:.3f} ' - 'Parallel: {:.3f} Other: {:.3f}'.format( - self.name, self.weighted, self.correct, self.fifth, - self.relative, self.relative_of_fifth, self.parallel, self.other)) + 'Parallel: {:.3f} ' + 'Other: {:.3f}'.format(self.name, self.weighted, self.correct, + self.fifth, self.relative, + self.relative_of_fifth, self.parallel, + self.other)) def add_parser(parser): From 3aae767c695a0647051dd1186ec97a68e35212dd Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 30 Nov 2018 17:18:40 -0500 Subject: [PATCH 07/21] pep8 fixes (take 2) --- madmom/evaluation/key.py | 34 +++++++++++++++++----------------- tests/test_evaluation_key.py | 19 +++++++++++++------ 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index 001cb7057..b76b07023 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -145,29 +145,29 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): elif not strict_fifth and \ (det_mode == ann_mode and ((det_root - ann_root) % 12 == 5)): error_type = 'fifth' - elif (ann_mode == major and det_mode != ann_mode - and ((det_root - ann_root) % 12 == 9)): + elif (ann_mode == major and det_mode != ann_mode and + ((det_root - ann_root) % 12 == 9)): error_type = 'relative' - elif (ann_mode == minor and det_mode != ann_mode - and ((det_root - ann_root) % 12 == 3)): + elif (ann_mode == minor and det_mode != ann_mode and + ((det_root - ann_root) % 12 == 3)): error_type = 'relative' elif relative_of_fifth \ - and (ann_mode == major and det_mode != ann_mode - and ((det_root - ann_root) % 12 == 4)): + and (ann_mode == major and det_mode != ann_mode and + ((det_root - ann_root) % 12 == 4)): error_type = 'relative_of_fifth' elif not strict_fifth and \ - (relative_of_fifth and (ann_mode == major - and det_mode != ann_mode - and ((det_root - ann_root) % 12 == 2))): + (relative_of_fifth and (ann_mode == major and + det_mode != ann_mode and + ((det_root - ann_root) % 12 == 2))): error_type = 'relative_of_fifth' - elif relative_of_fifth \ - and (ann_mode == minor and det_mode != ann_mode - and ((det_root - ann_root) % 12 == 10)): + elif relative_of_fifth and \ + (ann_mode == minor and det_mode != ann_mode and + ((det_root - ann_root) % 12 == 10)): error_type = 'relative_of_fifth' elif not strict_fifth and \ - (relative_of_fifth and (ann_mode == minor - and det_mode != ann_mode - and ((det_root - ann_root) % 12 == 8))): + (relative_of_fifth and (ann_mode == minor and + det_mode != ann_mode and + ((det_root - ann_root) % 12 == 8))): error_type = 'relative_of_fifth' elif det_mode != ann_mode and det_root == ann_root: error_type = 'parallel' @@ -285,9 +285,9 @@ def __init__(self, eval_objects, name=None): def _check_error_scores(self, eval_objects): all_the_same = True indScores = 0 - while (indScores < len(eval_objects)-1) and all_the_same: + while (indScores < len(eval_objects) - 1) and all_the_same: if eval_objects[indScores].error_scores \ - != eval_objects[indScores+1].error_scores: + != eval_objects[indScores + 1].error_scores: all_the_same = False break else: diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index 7859d6a91..48071058c 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -62,12 +62,16 @@ def test_values(self): class TestErrorTypeFunction(unittest.TestCase): def _compare_scores(self, correct, fifth_strict, fifth_lax, relative, - relative_of_fifth_up, relative_of_fifth_down, parallel): + relative_of_fifth_up, relative_of_fifth_down, + parallel): for det_key in range(24): cat = error_type(det_key, correct) cat_st = error_type(det_key, correct, strict_fifth=True) cat_rf = error_type(det_key, correct, relative_of_fifth=True) - cat_st_rf = error_type(det_key, correct, strict_fifth=True, relative_of_fifth=True) + cat_st_rf = error_type(det_key, + correct, + strict_fifth=True, + relative_of_fifth=True) if det_key == correct: self.assertEqual(cat, 'correct') self.assertEqual(cat_st, cat) @@ -204,7 +208,8 @@ def test_results(self): self.assertEqual(self.eval_relative.error_category, 'relative') self.assertEqual(self.eval_relative.score, 0.3) # Relative of Fifth - self.assertEqual(self.eval_relative_of_fifth.error_category, 'relative_of_fifth') + self.assertEqual(self.eval_relative_of_fifth.error_category, + 'relative_of_fifth') self.assertEqual(self.eval_relative_of_fifth.score, 0.0) # Parallel self.assertEqual(self.eval_parallel.error_category, 'parallel') @@ -247,10 +252,11 @@ def setUp(self): load_key(join(ANNOTATIONS_PATH, 'dummy.key')), name='eval_correct_different_scores' ) - self.eval_different_scores.error_scores={'correct':0.5} + self.eval_different_scores.error_scores = {'correct': 0.5} def test_check_error_scores(self): - evals = [self.eval_correct, self.eval_parallel, self.eval_different_scores, self.eval_other] + evals = [self.eval_correct, self.eval_parallel, + self.eval_different_scores, self.eval_other] with self.assertRaises(ValueError): KeyMeanEvaluation(evals) @@ -259,7 +265,8 @@ def test_empty_eval_list(self): KeyMeanEvaluation([]) def test_mean_results(self): - evals = [self.eval_correct, self.eval_parallel, self.eval_relative, self.eval_other] + evals = [self.eval_correct, self.eval_parallel, self.eval_relative, + self.eval_other] mean_eval = KeyMeanEvaluation(evals) self.assertAlmostEqual(mean_eval.correct, 1.0 / len(evals)) self.assertAlmostEqual(mean_eval.fifth, 0.0) From 0ceb7511ea7fc038438cf9742cb2cb987b15f88f Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Mon, 3 Dec 2018 11:36:44 -0500 Subject: [PATCH 08/21] attempt at refactoring the error_types function --- madmom/evaluation/key.py | 83 +++++++++++++++++++----------------- tests/test_evaluation_key.py | 47 +++++++++++++++++--- 2 files changed, 84 insertions(+), 46 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index b76b07023..8363ac6b8 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -59,6 +59,23 @@ def key_label_to_class(key_label): return key_class +def key_class_to_root_and_mode(key_class): + """ + Extract the root and mode from a key class id + :param key_class: number + :type key_class: int + :return: root id in terms of semi-tones apart from C and + mode id (0: major; 1: minor) + :rtype: tuple(int, int) + """ + if 0 <= key_class <= 23: + root = key_class % 12 + mode = key_class // 12 + else: + raise ValueError("{} is outside the [0; 23] range]".format(key_class)) + return root, mode + + def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): """ Compute the error category for a predicted key compared to @@ -132,48 +149,36 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): >>> error_type(14, 0, relative_of_fifth=True, strict_fifth=True) 'other' """ - ann_root = ann_key % 12 - ann_mode = ann_key // 12 - det_root = det_key % 12 - det_mode = det_key // 12 + (ann_root, ann_mode) = key_class_to_root_and_mode(ann_key) + (det_root, det_mode) = key_class_to_root_and_mode(det_key) major, minor = 0, 1 - if det_root == ann_root and det_mode == ann_mode: - error_type = 'correct' - elif det_mode == ann_mode and ((det_root - ann_root) % 12 == 7): - error_type = 'fifth' - elif not strict_fifth and \ - (det_mode == ann_mode and ((det_root - ann_root) % 12 == 5)): - error_type = 'fifth' - elif (ann_mode == major and det_mode != ann_mode and - ((det_root - ann_root) % 12 == 9)): - error_type = 'relative' - elif (ann_mode == minor and det_mode != ann_mode and - ((det_root - ann_root) % 12 == 3)): - error_type = 'relative' - elif relative_of_fifth \ - and (ann_mode == major and det_mode != ann_mode and - ((det_root - ann_root) % 12 == 4)): - error_type = 'relative_of_fifth' - elif not strict_fifth and \ - (relative_of_fifth and (ann_mode == major and - det_mode != ann_mode and - ((det_root - ann_root) % 12 == 2))): - error_type = 'relative_of_fifth' - elif relative_of_fifth and \ - (ann_mode == minor and det_mode != ann_mode and - ((det_root - ann_root) % 12 == 10)): - error_type = 'relative_of_fifth' - elif not strict_fifth and \ - (relative_of_fifth and (ann_mode == minor and - det_mode != ann_mode and - ((det_root - ann_root) % 12 == 8))): - error_type = 'relative_of_fifth' - elif det_mode != ann_mode and det_root == ann_root: - error_type = 'parallel' - else: - error_type = 'other' + root_distance = (det_root - ann_root) % 12 + error_type = 'other' + + if det_mode == ann_mode: + # Same modes ... + if det_root == ann_root: + error_type = 'correct' + if root_distance == 7 or (root_distance == 5 and not strict_fifth): + error_type = 'fifth' + else: + # Different modes ... + if det_root == ann_root: + error_type = 'parallel' + if (ann_mode == major and root_distance == 9) or \ + (ann_mode == minor and root_distance == 3): + error_type = 'relative' + if relative_of_fifth: + if ann_mode == major: + if (root_distance == 4) or \ + (root_distance == 2 and not strict_fifth): + error_type = 'relative_of_fifth' + if ann_mode == minor: + if (root_distance == 10) or \ + (root_distance == 8 and not strict_fifth): + error_type = 'relative_of_fifth' return error_type diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index 48071058c..cb4fb9711 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -59,11 +59,44 @@ def test_values(self): key_label_to_class('F# major')) +class TestKeyClassToRootAndModeFunction(unittest.TestCase): + + def test_values(self): + self.assertEqual(key_class_to_root_and_mode(0), (0, 0)) + self.assertEqual(key_class_to_root_and_mode(1), (1, 0)) + self.assertEqual(key_class_to_root_and_mode(2), (2, 0)) + self.assertEqual(key_class_to_root_and_mode(3), (3, 0)) + self.assertEqual(key_class_to_root_and_mode(4), (4, 0)) + self.assertEqual(key_class_to_root_and_mode(5), (5, 0)) + self.assertEqual(key_class_to_root_and_mode(6), (6, 0)) + self.assertEqual(key_class_to_root_and_mode(7), (7, 0)) + self.assertEqual(key_class_to_root_and_mode(8), (8, 0)) + self.assertEqual(key_class_to_root_and_mode(9), (9, 0)) + self.assertEqual(key_class_to_root_and_mode(10), (10, 0)) + self.assertEqual(key_class_to_root_and_mode(11), (11, 0)) + self.assertEqual(key_class_to_root_and_mode(12), (0, 1)) + self.assertEqual(key_class_to_root_and_mode(13), (1, 1)) + self.assertEqual(key_class_to_root_and_mode(14), (2, 1)) + self.assertEqual(key_class_to_root_and_mode(15), (3, 1)) + self.assertEqual(key_class_to_root_and_mode(16), (4, 1)) + self.assertEqual(key_class_to_root_and_mode(17), (5, 1)) + self.assertEqual(key_class_to_root_and_mode(18), (6, 1)) + self.assertEqual(key_class_to_root_and_mode(19), (7, 1)) + self.assertEqual(key_class_to_root_and_mode(20), (8, 1)) + self.assertEqual(key_class_to_root_and_mode(21), (9, 1)) + self.assertEqual(key_class_to_root_and_mode(22), (10, 1)) + self.assertEqual(key_class_to_root_and_mode(23), (11, 1)) + with self.assertRaises(ValueError): + key_class_to_root_and_mode(-4) + with self.assertRaises(ValueError): + key_class_to_root_and_mode(24) + + class TestErrorTypeFunction(unittest.TestCase): - def _compare_scores(self, correct, fifth_strict, fifth_lax, relative, - relative_of_fifth_up, relative_of_fifth_down, - parallel): + def _compare_error_types(self, correct, fifth_strict, fifth_lax, relative, + relative_of_fifth_up, relative_of_fifth_down, + parallel): for det_key in range(24): cat = error_type(det_key, correct) cat_st = error_type(det_key, correct, strict_fifth=True) @@ -109,7 +142,7 @@ def _compare_scores(self, correct, fifth_strict, fifth_lax, relative, self.assertEqual(cat_st_rf, cat) def test_values(self): - self._compare_scores( + self._compare_error_types( correct=key_label_to_class('c maj'), fifth_strict=key_label_to_class('g maj'), fifth_lax=key_label_to_class('f maj'), @@ -119,7 +152,7 @@ def test_values(self): parallel=key_label_to_class('c min') ) - self._compare_scores( + self._compare_error_types( correct=key_label_to_class('eb maj'), fifth_strict=key_label_to_class('bb maj'), fifth_lax=key_label_to_class('ab maj'), @@ -129,7 +162,7 @@ def test_values(self): parallel=key_label_to_class('eb min') ) - self._compare_scores( + self._compare_error_types( correct=key_label_to_class('a min'), fifth_strict=key_label_to_class('e min'), fifth_lax=key_label_to_class('d min'), @@ -139,7 +172,7 @@ def test_values(self): parallel=key_label_to_class('a maj') ) - self._compare_scores( + self._compare_error_types( correct=key_label_to_class('b min'), fifth_strict=key_label_to_class('gb min'), fifth_lax=key_label_to_class('e min'), From 2ede2f5ee5caa9db8ab22f7768a578df42eda171 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Mon, 3 Dec 2018 12:11:52 -0500 Subject: [PATCH 09/21] creating separate methods for each type of error --- madmom/evaluation/key.py | 75 ++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index 8363ac6b8..ef7be4a4f 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -76,6 +76,44 @@ def key_class_to_root_and_mode(key_class): return root, mode +def _compute_root_distance(det_root, ann_root): + return (det_root - ann_root) % 12 + + +def _is_correct(det_root, det_mode, ann_root, ann_mode): + return det_mode == ann_mode and det_root == ann_root + + +def _is_fifth(det_root, det_mode, ann_root, ann_mode, strict_fifth): + root_distance = _compute_root_distance(det_root, ann_root) + return det_mode == ann_mode and (root_distance == 7 or + (root_distance == 5 and not strict_fifth)) + + +def _is_parallel(det_root, det_mode, ann_root, ann_mode): + return det_root == ann_root and det_mode != ann_mode + + +def _is_relative(det_root, det_mode, ann_root, ann_mode, major, minor): + root_distance = _compute_root_distance(det_root, ann_root) + ann_mode_is_major = (ann_mode == major and root_distance == 9) + ann_mode_is_minor = (ann_mode == minor and root_distance == 3) + return det_mode != ann_mode and (ann_mode_is_major or ann_mode_is_minor) + + +def _is_relative_of_fifth(det_root, det_mode, ann_root, ann_mode, major, minor, + strict_fifth, relative_of_fifth): + root_distance = _compute_root_distance(det_root, ann_root) + ann_mode_is_major = ann_mode == major and ((root_distance == 4) or + (root_distance == 2 and + not strict_fifth)) + ann_mode_is_minor = ann_mode == minor and ((root_distance == 10) or + (root_distance == 8 and + not strict_fifth)) + return ann_mode != det_mode and relative_of_fifth and (ann_mode_is_major or + ann_mode_is_minor) + + def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): """ Compute the error category for a predicted key compared to @@ -153,32 +191,19 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): (det_root, det_mode) = key_class_to_root_and_mode(det_key) major, minor = 0, 1 - root_distance = (det_root - ann_root) % 12 - - error_type = 'other' - - if det_mode == ann_mode: - # Same modes ... - if det_root == ann_root: - error_type = 'correct' - if root_distance == 7 or (root_distance == 5 and not strict_fifth): - error_type = 'fifth' + if _is_correct(det_root, det_mode, ann_root, ann_mode): + error_type = 'correct' + elif _is_fifth(det_root, det_mode, ann_root, ann_mode, strict_fifth): + error_type = 'fifth' + elif _is_parallel(det_root, det_mode, ann_root, ann_mode): + error_type = 'parallel' + elif _is_relative(det_root, det_mode, ann_root, ann_mode, major, minor): + error_type = 'relative' + elif _is_relative_of_fifth(det_root, det_mode, ann_root, ann_mode, major, + minor, strict_fifth, relative_of_fifth): + error_type = 'relative_of_fifth' else: - # Different modes ... - if det_root == ann_root: - error_type = 'parallel' - if (ann_mode == major and root_distance == 9) or \ - (ann_mode == minor and root_distance == 3): - error_type = 'relative' - if relative_of_fifth: - if ann_mode == major: - if (root_distance == 4) or \ - (root_distance == 2 and not strict_fifth): - error_type = 'relative_of_fifth' - if ann_mode == minor: - if (root_distance == 10) or \ - (root_distance == 8 and not strict_fifth): - error_type = 'relative_of_fifth' + error_type = 'other' return error_type From b3050194c9cff425d0a596019bea69c5f650db0d Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Mon, 3 Dec 2018 12:24:12 -0500 Subject: [PATCH 10/21] decreasing number of arguments for _is_relative_of_fifth --- madmom/evaluation/key.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index ef7be4a4f..8ec630f88 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -80,29 +80,39 @@ def _compute_root_distance(det_root, ann_root): return (det_root - ann_root) % 12 -def _is_correct(det_root, det_mode, ann_root, ann_mode): +def _is_correct(det_key, ann_key): + det_root, det_mode = key_class_to_root_and_mode(det_key) + ann_root, ann_mode = key_class_to_root_and_mode(ann_key) return det_mode == ann_mode and det_root == ann_root -def _is_fifth(det_root, det_mode, ann_root, ann_mode, strict_fifth): +def _is_fifth(det_key, ann_key, strict_fifth): + det_root, det_mode = key_class_to_root_and_mode(det_key) + ann_root, ann_mode = key_class_to_root_and_mode(ann_key) root_distance = _compute_root_distance(det_root, ann_root) return det_mode == ann_mode and (root_distance == 7 or (root_distance == 5 and not strict_fifth)) -def _is_parallel(det_root, det_mode, ann_root, ann_mode): +def _is_parallel(det_key, ann_key): + det_root, det_mode = key_class_to_root_and_mode(det_key) + ann_root, ann_mode = key_class_to_root_and_mode(ann_key) return det_root == ann_root and det_mode != ann_mode -def _is_relative(det_root, det_mode, ann_root, ann_mode, major, minor): +def _is_relative(det_key, ann_key, major, minor): + det_root, det_mode = key_class_to_root_and_mode(det_key) + ann_root, ann_mode = key_class_to_root_and_mode(ann_key) root_distance = _compute_root_distance(det_root, ann_root) ann_mode_is_major = (ann_mode == major and root_distance == 9) ann_mode_is_minor = (ann_mode == minor and root_distance == 3) return det_mode != ann_mode and (ann_mode_is_major or ann_mode_is_minor) -def _is_relative_of_fifth(det_root, det_mode, ann_root, ann_mode, major, minor, - strict_fifth, relative_of_fifth): +def _is_relative_of_fifth(det_key, ann_key, major, minor, strict_fifth, + relative_of_fifth): + det_root, det_mode = key_class_to_root_and_mode(det_key) + ann_root, ann_mode = key_class_to_root_and_mode(ann_key) root_distance = _compute_root_distance(det_root, ann_root) ann_mode_is_major = ann_mode == major and ((root_distance == 4) or (root_distance == 2 and @@ -187,20 +197,18 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): >>> error_type(14, 0, relative_of_fifth=True, strict_fifth=True) 'other' """ - (ann_root, ann_mode) = key_class_to_root_and_mode(ann_key) - (det_root, det_mode) = key_class_to_root_and_mode(det_key) major, minor = 0, 1 - if _is_correct(det_root, det_mode, ann_root, ann_mode): + if _is_correct(det_key, ann_key): error_type = 'correct' - elif _is_fifth(det_root, det_mode, ann_root, ann_mode, strict_fifth): + elif _is_fifth(det_key, ann_key, strict_fifth): error_type = 'fifth' - elif _is_parallel(det_root, det_mode, ann_root, ann_mode): + elif _is_parallel(det_key, ann_key): error_type = 'parallel' - elif _is_relative(det_root, det_mode, ann_root, ann_mode, major, minor): + elif _is_relative(det_key, ann_key, major, minor): error_type = 'relative' - elif _is_relative_of_fifth(det_root, det_mode, ann_root, ann_mode, major, - minor, strict_fifth, relative_of_fifth): + elif _is_relative_of_fifth(det_key, ann_key, major, minor, strict_fifth, + relative_of_fifth): error_type = 'relative_of_fifth' else: error_type = 'other' From e5659c891cb023672d4bcfb97807ce875748a6d3 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Mon, 3 Dec 2018 12:30:33 -0500 Subject: [PATCH 11/21] nth refactor --- madmom/evaluation/key.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index 8ec630f88..826ac7d5c 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -114,14 +114,20 @@ def _is_relative_of_fifth(det_key, ann_key, major, minor, strict_fifth, det_root, det_mode = key_class_to_root_and_mode(det_key) ann_root, ann_mode = key_class_to_root_and_mode(ann_key) root_distance = _compute_root_distance(det_root, ann_root) + distance_criterion = _compute_relative_of_fifth_distance_criterion( + root_distance, ann_mode, strict_fifth, minor, major) + return ann_mode != det_mode and relative_of_fifth and distance_criterion + + +def _compute_relative_of_fifth_distance_criterion(root_distance, ann_mode, + strict_fifth, minor, major): ann_mode_is_major = ann_mode == major and ((root_distance == 4) or (root_distance == 2 and not strict_fifth)) ann_mode_is_minor = ann_mode == minor and ((root_distance == 10) or (root_distance == 8 and not strict_fifth)) - return ann_mode != det_mode and relative_of_fifth and (ann_mode_is_major or - ann_mode_is_minor) + return ann_mode_is_major or ann_mode_is_minor def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): From fe6f6d2b7d2b37af1da056ec1733d5e1e1201e58 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 25 Oct 2019 09:23:19 -0400 Subject: [PATCH 12/21] code review (quick stuff) --- madmom/evaluation/key.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index 826ac7d5c..ee568b821 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -175,11 +175,11 @@ def error_type(det_key, ann_key, strict_fifth=False, relative_of_fifth=False): 'correct' # annotated: 'C major' / detected: 'G major': +7 semitones - >>> error_type(7,0) + >>> error_type(7, 0) 'fifth' # annotated: 'C major' / detected: 'F major': -7 semitones (modulo 12) - >>> error_type(5,0) + >>> error_type(5, 0) 'fifth' # annotated: 'C major' / detected: 'F major': -7 semitones (modulo 12), @@ -228,11 +228,13 @@ class KeyEvaluation(EvaluationMixin): Parameters ---------- detection : str - File containing detected key + File containing detected key. annotation : str - File containing annotated key + File containing annotated key. strict_fifth : bool, optional Use strict interpretation of the 'fifth' category, as in MIREX. + relative_of_fifth: bool, optional + Consider relative of the fifth in the evaluation. name : str, optional Name of the evaluation object (e.g., the name of the song). @@ -263,7 +265,7 @@ def __init__(self, detection, self.annotation, strict_fifth, relative_of_fifth) - self.score = self.error_scores.get(self.error_category, 0.0) + self.score = self.error_scores[self.error_category] def tostring(self, **kwargs): """ From 33cc362f378e1a5ebba20b246075dde8e50428c9 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 25 Oct 2019 12:43:11 -0400 Subject: [PATCH 13/21] fixing tostring to only display info about relative of fifth when it is actually present in evaluation objects --- madmom/evaluation/key.py | 26 +++++++++++++------ tests/test_evaluation_key.py | 49 ++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index ee568b821..ab3b7bb33 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -307,7 +307,7 @@ class KeyMeanEvaluation(EvaluationMixin): def __init__(self, eval_objects, name=None): self.name = name or 'mean for {:d} files'.format(len(eval_objects)) - + self.relative_of_fifth_present = False n = len(eval_objects) if n > 0: c = Counter(e.error_category for e in eval_objects) @@ -325,6 +325,11 @@ def __init__(self, eval_objects, name=None): else: raise ValueError('Different error_scores found in the ' 'KeyEvaluation objects.') + + # Flag used in `tostring` to figure out whether to display info + # related to the relative of fifth + if self.relative_of_fifth > 0: + self.relative_of_fifth_present = True else: raise ValueError('The list of evaluations is empty.') @@ -341,13 +346,18 @@ def _check_error_scores(self, eval_objects): return all_the_same def tostring(self, **kwargs): - return ('{}\n Weighted: {:.3f} Correct: {:.3f} Fifth: {:.3f} ' - 'Relative: {:.3f} Relative of Fifth: {:.3f} ' - 'Parallel: {:.3f} ' - 'Other: {:.3f}'.format(self.name, self.weighted, self.correct, - self.fifth, self.relative, - self.relative_of_fifth, self.parallel, - self.other)) + ret = '' + spacing = ' ' * 2 + ret += '{}\n'.format(self.name) + spacing + ret += 'Weighted: {:.3f}'.format(self.weighted) + spacing + ret += 'Correct: {:.3f}'.format(self.correct) + spacing + ret += 'Fifth: {:.3f}'.format(self.fifth) + spacing + ret += 'Relative: {:.3f}'.format(self.relative) + spacing + if self.relative_of_fifth_present: + ret += 'Relative of fifth: {:.3f}'.format(self.relative_of_fifth) + spacing + ret += 'Parallel: {:.3f}'.format(self.parallel) + spacing + ret += 'Other: {:.3f}'.format(self.other) + return ret def add_parser(parser): diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index cb4fb9711..17c87343c 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -287,6 +287,22 @@ def setUp(self): ) self.eval_different_scores.error_scores = {'correct': 0.5} + self.eval_rel_of_fifth = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.relative_of_fifth.key.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + relative_of_fifth=True, + name='eval_rel_of_fifth' + ) + self.eval_rel_of_fifth.error_scores = {'relative_of_fifth': 0.7} + + self.eval_correct_w_rel_of_fifth = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.correct.key.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + relative_of_fifth=True, + name='eval_correct_w_rel_of_fifth' + ) + self.eval_correct_w_rel_of_fifth.error_scores = {'relative_of_fifth': 0.7} + def test_check_error_scores(self): evals = [self.eval_correct, self.eval_parallel, self.eval_different_scores, self.eval_other] @@ -300,7 +316,9 @@ def test_empty_eval_list(self): def test_mean_results(self): evals = [self.eval_correct, self.eval_parallel, self.eval_relative, self.eval_other] + mean_eval = KeyMeanEvaluation(evals) + self.assertAlmostEqual(mean_eval.correct, 1.0 / len(evals)) self.assertAlmostEqual(mean_eval.fifth, 0.0) self.assertAlmostEqual(mean_eval.relative, 1.0 / len(evals)) @@ -308,6 +326,37 @@ def test_mean_results(self): self.assertAlmostEqual(mean_eval.parallel, 1.0 / len(evals)) self.assertAlmostEqual(mean_eval.other, 1.0 / len(evals)) self.assertAlmostEqual(mean_eval.weighted, 0.375) + self.assertEqual(mean_eval.tostring(), + 'mean for 4 files\n ' + 'Weighted: 0.375 ' + 'Correct: 0.250 ' + 'Fifth: 0.000 ' + 'Relative: 0.250 ' + 'Parallel: 0.250 ' + 'Other: 0.250') + + def test_mean_results_w_rel_of_fifth(self): + evals = [self.eval_correct_w_rel_of_fifth, + self.eval_rel_of_fifth] + + mean_eval = KeyMeanEvaluation(evals, name='Jean-Guy') + + self.assertAlmostEqual(mean_eval.correct, 1.0 / len(evals)) + self.assertAlmostEqual(mean_eval.fifth, 0.0) + self.assertAlmostEqual(mean_eval.relative, 0.0) + self.assertAlmostEqual(mean_eval.relative_of_fifth, 1.0 / len(evals)) + self.assertAlmostEqual(mean_eval.parallel, 0.0 / len(evals)) + self.assertAlmostEqual(mean_eval.other, 0.0 / len(evals)) + self.assertAlmostEqual(mean_eval.weighted, 0.5) + self.assertEqual(mean_eval.tostring(), + 'Jean-Guy\n ' + 'Weighted: 0.500 ' + 'Correct: 0.500 ' + 'Fifth: 0.000 ' + 'Relative: 0.000 ' + 'Relative of fifth: 0.500 ' + 'Parallel: 0.000 ' + 'Other: 0.000') class TestAddParserFunction(unittest.TestCase): From 0e451f59f1864f7de8f5ea40fd98fd8ac76c5daf Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 25 Oct 2019 15:26:17 -0400 Subject: [PATCH 14/21] Really checking that all key evaluation objects are configured in the same way --- madmom/evaluation/key.py | 79 ++++++++++++++++++++++-------------- tests/test_evaluation_key.py | 16 +++----- 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index ab3b7bb33..23186c042 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -261,10 +261,12 @@ def __init__(self, detection, self.name = name or '' self.detection = key_label_to_class(detection) self.annotation = key_label_to_class(annotation) + self.strict_fifth = strict_fifth + self.relative_of_fifth = relative_of_fifth self.error_category = error_type(self.detection, self.annotation, - strict_fifth, - relative_of_fifth) + self.strict_fifth, + self.relative_of_fifth) self.score = self.error_scores[self.error_category] def tostring(self, **kwargs): @@ -310,41 +312,28 @@ def __init__(self, eval_objects, name=None): self.relative_of_fifth_present = False n = len(eval_objects) if n > 0: - c = Counter(e.error_category for e in eval_objects) - - self.correct = float(c['correct']) / n - self.fifth = float(c['fifth']) / n - self.relative = float(c['relative']) / n - self.relative_of_fifth = float(c['relative_of_fifth']) / n - self.parallel = float(c['parallel']) / n - self.other = float(c['other']) / n - # Check that all the error_scores in the eval_objects are the same - # (otherwise a weighted result would be hard to interpret) - if self._check_error_scores(eval_objects): + # Check that all the key evaluation objects are evaluating errors + # the same way + if check_key_eval_objects(eval_objects): + # Counting the error categories + c = Counter(e.error_category for e in eval_objects) + self.correct = float(c['correct']) / n + self.fifth = float(c['fifth']) / n + self.relative = float(c['relative']) / n + self.parallel = float(c['parallel']) / n + self.other = float(c['other']) / n + self.relative_of_fifth = float(c['relative_of_fifth']) / n self.weighted = sum(e.score for e in eval_objects) / n + + if self.relative_of_fifth > 0: + self.relative_of_fifth_present = True else: - raise ValueError('Different error_scores found in the ' - 'KeyEvaluation objects.') + raise ValueError('The KeyEvaluation objects are not ' + 'all the same.') - # Flag used in `tostring` to figure out whether to display info - # related to the relative of fifth - if self.relative_of_fifth > 0: - self.relative_of_fifth_present = True else: raise ValueError('The list of evaluations is empty.') - def _check_error_scores(self, eval_objects): - all_the_same = True - indScores = 0 - while (indScores < len(eval_objects) - 1) and all_the_same: - if eval_objects[indScores].error_scores \ - != eval_objects[indScores + 1].error_scores: - all_the_same = False - break - else: - indScores += 1 - return all_the_same - def tostring(self, **kwargs): ret = '' spacing = ' ' * 2 @@ -360,6 +349,34 @@ def tostring(self, **kwargs): return ret +def check_key_eval_objects(key_eval_objects): + """ + Check whether all the key evaluation objects in a list have the same way of + scoring errors, evaluating fifth and if they include relative of fifth + + Parameters: + ---------- + key_eval_objects: list + Key evaluation objects + + :return: + """ + all_the_same = True + ind_eval = 0 + while (ind_eval < len(key_eval_objects) - 1) and all_the_same: + if key_eval_objects[ind_eval].error_scores \ + != key_eval_objects[ind_eval + 1].error_scores or \ + key_eval_objects[ind_eval].strict_fifth \ + != key_eval_objects[ind_eval + 1].strict_fifth or \ + key_eval_objects[ind_eval].relative_of_fifth \ + != key_eval_objects[ind_eval + 1].relative_of_fifth: + all_the_same = False + break + else: + ind_eval += 1 + return all_the_same + + def add_parser(parser): """ Add a key evaluation sub-parser to an existing parser. diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index 17c87343c..3a93af722 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -293,22 +293,18 @@ def setUp(self): relative_of_fifth=True, name='eval_rel_of_fifth' ) - self.eval_rel_of_fifth.error_scores = {'relative_of_fifth': 0.7} - self.eval_correct_w_rel_of_fifth = KeyEvaluation( - load_key(join(DETECTIONS_PATH, 'dummy.correct.key.txt')), - load_key(join(ANNOTATIONS_PATH, 'dummy.key')), - relative_of_fifth=True, - name='eval_correct_w_rel_of_fifth' - ) - self.eval_correct_w_rel_of_fifth.error_scores = {'relative_of_fifth': 0.7} - - def test_check_error_scores(self): + def test_check_key_eval_objects(self): evals = [self.eval_correct, self.eval_parallel, self.eval_different_scores, self.eval_other] with self.assertRaises(ValueError): KeyMeanEvaluation(evals) + evals = [self.eval_correct, self.eval_parallel, + self.eval_rel_of_fifth] + with self.assertRaises(ValueError): + KeyMeanEvaluation(evals) + def test_empty_eval_list(self): with self.assertRaises(ValueError): KeyMeanEvaluation([]) From 661adcd63e2e45846747aa67c6571fbc7efcfff7 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 25 Oct 2019 16:03:06 -0400 Subject: [PATCH 15/21] hopefully reducing the code complexity --- madmom/evaluation/key.py | 44 ++++++++++++++++++++++++++-------------- madmom/models | 2 +- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index 23186c042..a2200b872 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -315,25 +315,26 @@ def __init__(self, eval_objects, name=None): # Check that all the key evaluation objects are evaluating errors # the same way if check_key_eval_objects(eval_objects): - # Counting the error categories - c = Counter(e.error_category for e in eval_objects) - self.correct = float(c['correct']) / n - self.fifth = float(c['fifth']) / n - self.relative = float(c['relative']) / n - self.parallel = float(c['parallel']) / n - self.other = float(c['other']) / n - self.relative_of_fifth = float(c['relative_of_fifth']) / n - self.weighted = sum(e.score for e in eval_objects) / n + self._count_evaluations(eval_objects) if self.relative_of_fifth > 0: self.relative_of_fifth_present = True else: raise ValueError('The KeyEvaluation objects are not ' 'all the same.') - else: raise ValueError('The list of evaluations is empty.') + def _count_evaluations(self, eval_objects): + c = Counter(e.error_category for e in eval_objects) + self.correct = float(c['correct']) / n + self.fifth = float(c['fifth']) / n + self.relative = float(c['relative']) / n + self.parallel = float(c['parallel']) / n + self.other = float(c['other']) / n + self.relative_of_fifth = float(c['relative_of_fifth']) / n + self.weighted = sum(e.score for e in eval_objects) / n + def tostring(self, **kwargs): ret = '' spacing = ' ' * 2 @@ -361,15 +362,20 @@ def check_key_eval_objects(key_eval_objects): :return: """ + + error_scores_OK = check_error_scores(key_eval_objects) + strict_fifth_OK = check_strict_fifth(key_eval_objects) + rel_fifth_OK = check_relative_of_fifth(key_eval_objects) + + return error_scores_OK and strict_fifth_OK and rel_fifth_OK + + +def check_error_scores(key_eval_objects): all_the_same = True ind_eval = 0 while (ind_eval < len(key_eval_objects) - 1) and all_the_same: if key_eval_objects[ind_eval].error_scores \ - != key_eval_objects[ind_eval + 1].error_scores or \ - key_eval_objects[ind_eval].strict_fifth \ - != key_eval_objects[ind_eval + 1].strict_fifth or \ - key_eval_objects[ind_eval].relative_of_fifth \ - != key_eval_objects[ind_eval + 1].relative_of_fifth: + != key_eval_objects[ind_eval + 1].error_scores: all_the_same = False break else: @@ -377,6 +383,14 @@ def check_key_eval_objects(key_eval_objects): return all_the_same +def check_strict_fifth(key_eval_objects): + return len(set([e.strict_fifth for e in key_eval_objects])) == 1 + + +def check_relative_of_fifth(key_eval_objects): + return len(set([e.relative_of_fifth for e in key_eval_objects])) == 1 + + def add_parser(parser): """ Add a key evaluation sub-parser to an existing parser. diff --git a/madmom/models b/madmom/models index df793487a..31fe6ce52 160000 --- a/madmom/models +++ b/madmom/models @@ -1 +1 @@ -Subproject commit df793487a4cc9ea6473034a35b70624b55415de5 +Subproject commit 31fe6ce52affb794c2e3511908fbeccfd68f6925 From 5f05759cdb5707966ebdb79dc8ac9e493ef0c7de Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 25 Oct 2019 16:07:52 -0400 Subject: [PATCH 16/21] new attempt a diminishing complexity --- madmom/evaluation/key.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index a2200b872..33b07111d 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -310,22 +310,16 @@ class KeyMeanEvaluation(EvaluationMixin): def __init__(self, eval_objects, name=None): self.name = name or 'mean for {:d} files'.format(len(eval_objects)) self.relative_of_fifth_present = False - n = len(eval_objects) - if n > 0: - # Check that all the key evaluation objects are evaluating errors - # the same way - if check_key_eval_objects(eval_objects): - self._count_evaluations(eval_objects) - - if self.relative_of_fifth > 0: - self.relative_of_fifth_present = True - else: - raise ValueError('The KeyEvaluation objects are not ' - 'all the same.') + if check_key_eval_objects(eval_objects): + self._count_evaluations(eval_objects) + if self.relative_of_fifth > 0: + self.relative_of_fifth_present = True else: - raise ValueError('The list of evaluations is empty.') + raise ValueError('The KeyEvaluation objects are not ' + 'all the same.') def _count_evaluations(self, eval_objects): + n = len(eval_objects) c = Counter(e.error_category for e in eval_objects) self.correct = float(c['correct']) / n self.fifth = float(c['fifth']) / n @@ -359,14 +353,10 @@ def check_key_eval_objects(key_eval_objects): ---------- key_eval_objects: list Key evaluation objects - - :return: """ - error_scores_OK = check_error_scores(key_eval_objects) strict_fifth_OK = check_strict_fifth(key_eval_objects) rel_fifth_OK = check_relative_of_fifth(key_eval_objects) - return error_scores_OK and strict_fifth_OK and rel_fifth_OK From 2bc55ebe1075ed451e92c1a276dc2b84804f67d4 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 25 Oct 2019 16:13:31 -0400 Subject: [PATCH 17/21] fix line too long --- madmom/evaluation/key.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index 33b07111d..ec1f7f663 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -338,7 +338,8 @@ def tostring(self, **kwargs): ret += 'Fifth: {:.3f}'.format(self.fifth) + spacing ret += 'Relative: {:.3f}'.format(self.relative) + spacing if self.relative_of_fifth_present: - ret += 'Relative of fifth: {:.3f}'.format(self.relative_of_fifth) + spacing + ret += 'Relative of fifth: {:.3f}'.format(self.relative_of_fifth) \ + + spacing ret += 'Parallel: {:.3f}'.format(self.parallel) + spacing ret += 'Other: {:.3f}'.format(self.other) return ret From dfbf7a48add417b421894282b25fb1a1cf3c9239 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 25 Oct 2019 16:23:46 -0400 Subject: [PATCH 18/21] fix test --- tests/test_evaluation_key.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index 3a93af722..948e1fefd 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -287,6 +287,13 @@ def setUp(self): ) self.eval_different_scores.error_scores = {'correct': 0.5} + self.eval_correct_w_rel_of_fifth = KeyEvaluation( + load_key(join(DETECTIONS_PATH, 'dummy.correct.key.txt')), + load_key(join(ANNOTATIONS_PATH, 'dummy.key')), + relative_of_fifth=True, + name='eval_correct_w_rel_of_fifth' + ) + self.eval_rel_of_fifth = KeyEvaluation( load_key(join(DETECTIONS_PATH, 'dummy.relative_of_fifth.key.txt')), load_key(join(ANNOTATIONS_PATH, 'dummy.key')), From 2f9d65e5d6de0298c1f2929028945c1c825f0383 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Fri, 25 Oct 2019 16:38:21 -0400 Subject: [PATCH 19/21] update models --- madmom/models | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/madmom/models b/madmom/models index 31fe6ce52..df793487a 160000 --- a/madmom/models +++ b/madmom/models @@ -1 +1 @@ -Subproject commit 31fe6ce52affb794c2e3511908fbeccfd68f6925 +Subproject commit df793487a4cc9ea6473034a35b70624b55415de5 From d6059d10130d9f520932f7c157658868cade5587 Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Mon, 10 Jan 2022 15:36:12 -0500 Subject: [PATCH 20/21] simplification of key eval objects checking + more flexible way of adding relative of fifth error score --- madmom/evaluation/key.py | 51 +++++++++++++----------------------- tests/test_evaluation_key.py | 1 - 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/madmom/evaluation/key.py b/madmom/evaluation/key.py index ec1f7f663..18f573c88 100644 --- a/madmom/evaluation/key.py +++ b/madmom/evaluation/key.py @@ -309,11 +309,8 @@ class KeyMeanEvaluation(EvaluationMixin): def __init__(self, eval_objects, name=None): self.name = name or 'mean for {:d} files'.format(len(eval_objects)) - self.relative_of_fifth_present = False - if check_key_eval_objects(eval_objects): + if _check_key_eval_objects(eval_objects): self._count_evaluations(eval_objects) - if self.relative_of_fifth > 0: - self.relative_of_fifth_present = True else: raise ValueError('The KeyEvaluation objects are not ' 'all the same.') @@ -326,7 +323,10 @@ def _count_evaluations(self, eval_objects): self.relative = float(c['relative']) / n self.parallel = float(c['parallel']) / n self.other = float(c['other']) / n - self.relative_of_fifth = float(c['relative_of_fifth']) / n + if 'relative_of_fifth' in c.keys(): + self.relative_of_fifth = float(c['relative_of_fifth']) / n + else: + self.relative_of_fifth = None self.weighted = sum(e.score for e in eval_objects) / n def tostring(self, **kwargs): @@ -337,7 +337,7 @@ def tostring(self, **kwargs): ret += 'Correct: {:.3f}'.format(self.correct) + spacing ret += 'Fifth: {:.3f}'.format(self.fifth) + spacing ret += 'Relative: {:.3f}'.format(self.relative) + spacing - if self.relative_of_fifth_present: + if self.relative_of_fifth: ret += 'Relative of fifth: {:.3f}'.format(self.relative_of_fifth) \ + spacing ret += 'Parallel: {:.3f}'.format(self.parallel) + spacing @@ -345,41 +345,26 @@ def tostring(self, **kwargs): return ret -def check_key_eval_objects(key_eval_objects): +def _check_key_eval_objects(key_eval_objects): """ Check whether all the key evaluation objects in a list have the same way of - scoring errors, evaluating fifth and if they include relative of fifth + scoring errors Parameters: ---------- key_eval_objects: list Key evaluation objects """ - error_scores_OK = check_error_scores(key_eval_objects) - strict_fifth_OK = check_strict_fifth(key_eval_objects) - rel_fifth_OK = check_relative_of_fifth(key_eval_objects) - return error_scores_OK and strict_fifth_OK and rel_fifth_OK - - -def check_error_scores(key_eval_objects): - all_the_same = True - ind_eval = 0 - while (ind_eval < len(key_eval_objects) - 1) and all_the_same: - if key_eval_objects[ind_eval].error_scores \ - != key_eval_objects[ind_eval + 1].error_scores: - all_the_same = False - break - else: - ind_eval += 1 - return all_the_same - - -def check_strict_fifth(key_eval_objects): - return len(set([e.strict_fifth for e in key_eval_objects])) == 1 - - -def check_relative_of_fifth(key_eval_objects): - return len(set([e.relative_of_fifth for e in key_eval_objects])) == 1 + if len(key_eval_objects) > 0: + e = key_eval_objects[0] + for ke in key_eval_objects[1:]: + if (ke.error_scores != e.error_scores or + ke.strict_fifth != e.strict_fifth or + ke.relative_of_fifth != e.relative_of_fifth): + return False + return True + else: + raise ValueError('No KeyEvaluation objects to check.') def add_parser(parser): diff --git a/tests/test_evaluation_key.py b/tests/test_evaluation_key.py index 948e1fefd..534e39300 100644 --- a/tests/test_evaluation_key.py +++ b/tests/test_evaluation_key.py @@ -325,7 +325,6 @@ def test_mean_results(self): self.assertAlmostEqual(mean_eval.correct, 1.0 / len(evals)) self.assertAlmostEqual(mean_eval.fifth, 0.0) self.assertAlmostEqual(mean_eval.relative, 1.0 / len(evals)) - self.assertAlmostEqual(mean_eval.relative_of_fifth, 0.0) self.assertAlmostEqual(mean_eval.parallel, 1.0 / len(evals)) self.assertAlmostEqual(mean_eval.other, 1.0 / len(evals)) self.assertAlmostEqual(mean_eval.weighted, 0.375) From 04f6566046f86ebd3c302544e21605af2d7babaf Mon Sep 17 00:00:00 2001 From: Bertrand Scherrer Date: Mon, 10 Jan 2022 15:39:56 -0500 Subject: [PATCH 21/21] update models --- madmom/models | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/madmom/models b/madmom/models index df793487a..7e3dc1b0c 160000 --- a/madmom/models +++ b/madmom/models @@ -1 +1 @@ -Subproject commit df793487a4cc9ea6473034a35b70624b55415de5 +Subproject commit 7e3dc1b0cad499792767074d03c38b194b9b0a79