Test scripts for coverage analysis.

framework/test/jobs_coverage.pl

@@ -0,0 +1,77 @@
+#!/usr/bin/env perl
+#
+#-------------------------------------------------------------------------------
+# Copyright (c) 2014-2024 René Just, Darioush Jalali, and Defects4J contributors.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#-------------------------------------------------------------------------------
+
+=pod
+
+=head1 NAME
+
+jobs_mutation.pl -- output a list of invocations of the test_mutation_cmd.sh script, one line per bug.
+
+=head1 SYNOPSIS
+
+  jobs_mutation.pl | shuf | parallel -j20 --progress
+
+=head1 DESCRIPTION
+
+Determines all active bugs and outputs a list of invocations of the
+test_mutation_cmd.sh script -- one line per bug. The output list of
+jobs can be processed in parallel, e.g., using GNU parallel.
+
+=cut
+use warnings;
+use strict;
+
+use FindBin;
+use File::Basename;
+use Cwd qw(abs_path);
+use Getopt::Std;
+use Pod::Usage;
+
+use lib abs_path("$FindBin::Bin/../core");
+use Constants;
+use Project;
+
+#
+# Process arguments and issue usage message if necessary.
+#
+
+# Read all project modules
+opendir(my $dir, "$CORE_DIR/Project") or die "Cannot open directory: $!";
+my @files = readdir($dir);
+closedir($dir);
+
+# For each project module, instantiate project and determine its name and number
+# of bugs.
+my $total = 0;
+my @projects = ();
+for my $file (@files) {
+  $file =~ "^([^.]+)\.pm\$" or next;
+  my $pid=$1;
+  my $project = Project::create_project($pid);
+  my $name = $project->{prog_name};
+  my @bug_ids = $project->get_bug_ids();
+  for my $id (@bug_ids) {
+    print("./test_coverage_cmd.sh -p $pid -b $id \n");
+  }
+}

framework/test/test_coverage_cmd.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+################################################################################
+#
+# This script runs the defects4j mutation command for all bugs or a subset of
+# bugs for a given project.
+#
+# Examples for Lang:
+#   * All bugs:         ./test_mutation_cmd.sh -pLang
+#   * Bugs 1-10:        ./test_mutation_cmd.sh -pLang -b1..10
+#   * Bugs 1 and 3:     ./test_mutation_cmd.sh -pLang -b1 -b3
+#   * Bugs 1-10 and 20: ./test_mutation_cmd.sh -pLang -b1..10 -b20
+#
+################################################################################
+# Import helper subroutines and variables, and init Defects4J
+source test.include
+
+# Print usage message and exit
+usage() {
+    local known_pids; known_pids=$(defects4j pids)
+    echo "usage: $0 -p <project id> [-b <bug id> ... | -b <bug id range> ... ]"
+    echo "Project ids:"
+    for pid in $known_pids; do
+        echo "  * $pid"
+    done
+    exit 1
+}
+
+# Check arguments
+while getopts ":p:b:" opt; do
+    case $opt in
+        p) PID="$OPTARG"
+            ;;
+        b) if [[ "$OPTARG" =~ ^[0-9]*\.\.[0-9]*$ ]]; then
+                BUGS="$BUGS $(eval "echo {$OPTARG}")"
+           else
+               BUGS="$BUGS $OPTARG"
+           fi
+            ;;
+        \?)
+            echo "Unknown option: -$OPTARG" >&2
+            usage
+            ;;
+        :)
+            echo "No argument provided: -$OPTARG." >&2
+            usage
+            ;;
+  esac
+done
+
+if [ "$PID" == "" ]; then
+    usage
+fi
+
+if [ ! -e "$BASE_DIR/framework/core/Project/$PID.pm" ]; then
+    usage
+fi
+
+init
+
+# Run all bugs, unless otherwise specified
+if [ "$BUGS" == "" ]; then
+    BUGS="$(get_bug_ids "$BASE_DIR/framework/projects/$PID/$BUGS_CSV_ACTIVE")"
+fi
+
+# Create log file
+script_name_without_sh=${script//.sh/}
+LOG="$TEST_DIR/${script_name_without_sh}$(printf '_%s_%s' "$PID" $$).log"
+OUT_DIR="$TEST_DIR/${script_name_without_sh}$(printf '_%s_%s' "$PID" $$).cov"
+mkdir -p "$OUT_DIR"
+
+# Reproduce all bugs (and log all results), regardless of whether errors occur
+HALT_ON_ERROR=0
+
+test_dir=$(mktemp -d)
+for bid in $BUGS; do
+    # Skip all bug ids that do not exist in the active-bugs csv
+    if ! grep -q "^$bid," "$BASE_DIR/framework/projects/$PID/$BUGS_CSV_ACTIVE"; then
+        warn "Skipping bug ID that is not listed in active-bugs csv: $PID-$bid"
+        continue
+    fi
+
+    # Test mutation analysis for the fixed version only.
+    vid="${bid}f"
+    work_dir="$test_dir/$PID-$vid"
+    defects4j checkout -p "$PID" -v "$vid" -w "$work_dir" || die "checkout: $PID-$vid"
+    if defects4j coverage -w "$work_dir"; then
+      cat "$work_dir/summary.csv" > "$OUT_DIR/$bid.summary"
+    else 
+      echo "ERROR: $PID-$bid" > "$OUT_DIR/$PID-$bid.summary"
+    fi
+    cp "$work_dir/all_tests" "$OUT_DIR/$PID-$bid.tests"
+done
+rm -rf "$test_dir"
+HALT_ON_ERROR=1
+
+# Print a summary of what went wrong
+if [ $ERROR != 0 ]; then
+    printf '=%.s' $(seq 1 80) 1>&2
+    echo 1>&2
+    echo "The following errors occurred:" 1>&2
+    cat "$LOG" 1>&2
+fi
+
+# Indicate whether an error occurred
+exit $ERROR