diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py index 97264748f..bbc23a1fc 100644 --- a/scoring/score_submissions.py +++ b/scoring/score_submissions.py @@ -30,9 +30,11 @@ 'submission_directory', None, 'Path to submission directory containing experiment directories.') -flags.DEFINE_string('output_dir', - 'scoring_results', - 'Path to save performance profile table and plot.') +flags.DEFINE_string( + 'output_dir', + 'scoring_results', + 'Path to save performance profile artifacts, submission_summaries and results files.' +) flags.DEFINE_boolean('compute_performance_profiles', False, 'Whether or not to compute the performance profiles.') @@ -51,11 +53,16 @@ None, 'Filename to save the processed results that are fed into the performance profile functions.' ) -flags.DEFINE_boolean( +flags.DEFINE_string( 'load_results_from_filename', None, 'Filename to load processed results from that are fed into performance profile functions' ) +flags.DEFINE_string( + 'exclude_submissions', + '', + 'Optional comma seperated list of names of submissions to exclude from scoring.' +) FLAGS = flags.FLAGS @@ -129,6 +136,22 @@ def get_submission_summary(df, include_test_split=True): return df +def compute_leaderboard_score(df, normalize=False): + """Compute leaderboard score by taking integral of performance profile. + + Args: + df: pd.DataFrame returned from `compute_performance_profiles`. + normalize: divide by the range of the performance profile's tau. + + Returns: + pd.DataFrame with one column of scores indexed by submission. + """ + scores = np.trapz(df, x=df.columns) + if normalize: + scores /= df.columns.max() - df.columns.min() + return pd.DataFrame(scores, columns=['score'], index=df.index) + + def main(_): results = {} os.makedirs(FLAGS.output_dir, exist_ok=True) @@ -144,6 +167,8 @@ def main(_): for submission in os.listdir( os.path.join(FLAGS.submission_directory, team)): print(submission) + if submission in FLAGS.exclude_submissions.split(','): + continue experiment_path = os.path.join(FLAGS.submission_directory, team, submission) @@ -185,9 +210,13 @@ def main(_): os.mkdir(FLAGS.output_dir) performance_profile.plot_performance_profiles( performance_profile_df, 'score', save_dir=FLAGS.output_dir) - perf_df = tabulate( + performance_profile_str = tabulate( performance_profile_df.T, headers='keys', tablefmt='psql') - logging.info(f'Performance profile:\n {perf_df}') + logging.info(f'Performance profile:\n {performance_profile_str}') + scores = compute_leaderboard_score(performance_profile_df) + scores.to_csv(os.path.join(FLAGS.output_dir, 'scores.csv')) + scores_str = tabulate(scores, headers='keys', tablefmt='psql') + logging.info(f'Scores: \n {scores_str}') if __name__ == '__main__':