diff --git a/framework/test/jobs_coverage.pl b/framework/test/jobs_coverage.pl new file mode 100755 index 000000000..ec70c9d04 --- /dev/null +++ b/framework/test/jobs_coverage.pl @@ -0,0 +1,77 @@ +#!/usr/bin/env perl +# +#------------------------------------------------------------------------------- +# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +#------------------------------------------------------------------------------- + +=pod + +=head1 NAME + +jobs_mutation.pl -- output a list of invocations of the test_mutation_cmd.sh script, one line per bug. + +=head1 SYNOPSIS + + jobs_mutation.pl | shuf | parallel -j20 --progress + +=head1 DESCRIPTION + +Determines all active bugs and outputs a list of invocations of the +test_mutation_cmd.sh script -- one line per bug. The output list of +jobs can be processed in parallel, e.g., using GNU parallel. + +=cut +use warnings; +use strict; + +use FindBin; +use File::Basename; +use Cwd qw(abs_path); +use Getopt::Std; +use Pod::Usage; + +use lib abs_path("$FindBin::Bin/../core"); +use Constants; +use Project; + +# +# Process arguments and issue usage message if necessary. +# + +# Read all project modules +opendir(my $dir, "$CORE_DIR/Project") or die "Cannot open directory: $!"; +my @files = readdir($dir); +closedir($dir); + +# For each project module, instantiate project and determine its name and number +# of bugs. +my $total = 0; +my @projects = (); +for my $file (@files) { + $file =~ "^([^.]+)\.pm\$" or next; + my $pid=$1; + my $project = Project::create_project($pid); + my $name = $project->{prog_name}; + my @bug_ids = $project->get_bug_ids(); + for my $id (@bug_ids) { + print("./test_coverage_cmd.sh -p $pid -b $id \n"); + } +} diff --git a/framework/test/test_coverage_cmd.sh b/framework/test/test_coverage_cmd.sh new file mode 100755 index 000000000..b96f9af2a --- /dev/null +++ b/framework/test/test_coverage_cmd.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +################################################################################ +# +# This script runs the defects4j mutation command for all bugs or a subset of +# bugs for a given project. +# +# Examples for Lang: +# * All bugs: ./test_mutation_cmd.sh -pLang +# * Bugs 1-10: ./test_mutation_cmd.sh -pLang -b1..10 +# * Bugs 1 and 3: ./test_mutation_cmd.sh -pLang -b1 -b3 +# * Bugs 1-10 and 20: ./test_mutation_cmd.sh -pLang -b1..10 -b20 +# +################################################################################ +# Import helper subroutines and variables, and init Defects4J +source test.include + +# Print usage message and exit +usage() { + local known_pids; known_pids=$(defects4j pids) + echo "usage: $0 -p [-b ... | -b ... ]" + echo "Project ids:" + for pid in $known_pids; do + echo " * $pid" + done + exit 1 +} + +# Check arguments +while getopts ":p:b:" opt; do + case $opt in + p) PID="$OPTARG" + ;; + b) if [[ "$OPTARG" =~ ^[0-9]*\.\.[0-9]*$ ]]; then + BUGS="$BUGS $(eval "echo {$OPTARG}")" + else + BUGS="$BUGS $OPTARG" + fi + ;; + \?) + echo "Unknown option: -$OPTARG" >&2 + usage + ;; + :) + echo "No argument provided: -$OPTARG." >&2 + usage + ;; + esac +done + +if [ "$PID" == "" ]; then + usage +fi + +if [ ! -e "$BASE_DIR/framework/core/Project/$PID.pm" ]; then + usage +fi + +init + +# Run all bugs, unless otherwise specified +if [ "$BUGS" == "" ]; then + BUGS="$(get_bug_ids "$BASE_DIR/framework/projects/$PID/$BUGS_CSV_ACTIVE")" +fi + +# Create log file +script_name_without_sh=${script//.sh/} +LOG="$TEST_DIR/${script_name_without_sh}$(printf '_%s_%s' "$PID" $$).log" +OUT_DIR="$TEST_DIR/${script_name_without_sh}$(printf '_%s_%s' "$PID" $$).cov" +mkdir -p "$OUT_DIR" + +# Reproduce all bugs (and log all results), regardless of whether errors occur +HALT_ON_ERROR=0 + +test_dir=$(mktemp -d) +for bid in $BUGS; do + # Skip all bug ids that do not exist in the active-bugs csv + if ! grep -q "^$bid," "$BASE_DIR/framework/projects/$PID/$BUGS_CSV_ACTIVE"; then + warn "Skipping bug ID that is not listed in active-bugs csv: $PID-$bid" + continue + fi + + # Test mutation analysis for the fixed version only. + vid="${bid}f" + work_dir="$test_dir/$PID-$vid" + defects4j checkout -p "$PID" -v "$vid" -w "$work_dir" || die "checkout: $PID-$vid" + if defects4j coverage -w "$work_dir"; then + cat "$work_dir/summary.csv" > "$OUT_DIR/$bid.summary" + else + echo "ERROR: $PID-$bid" > "$OUT_DIR/$PID-$bid.summary" + fi + cp "$work_dir/all_tests" "$OUT_DIR/$PID-$bid.tests" +done +rm -rf "$test_dir" +HALT_ON_ERROR=1 + +# Print a summary of what went wrong +if [ $ERROR != 0 ]; then + printf '=%.s' $(seq 1 80) 1>&2 + echo 1>&2 + echo "The following errors occurred:" 1>&2 + cat "$LOG" 1>&2 +fi + +# Indicate whether an error occurred +exit $ERROR