Merge pull request #88 from udem-dlteam/laurent/more-benchmarks

More benchmarks

benchmarks/compile-times/run.sh

@@ -12,6 +12,12 @@ print_time()
   printf "%s %s\n" "$((ms/1000)).$((ms/100%10))$((ms/10%10))$((ms%10))s" "$2"
 }
 
+comp_program_options() { # $1 = program
+  case $1 in
+    "examples/c4.c") echo "-I./examples/c4-libs/" ;;
+  esac
+}
+
 compile() {
   # shell=$1
   file=$2
@@ -20,13 +26,13 @@ compile() {
   output_name=${4-$name}
 
   if [ $1 = "gcc" ]; then
-    TIME_MS=$(( `bash -c "time $COMP_DIR/pnut-sh-base.exe $file $options > $COMP_DIR/$output_name-with-$1.sh" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
+    TIME_MS=$(( `bash -c "time $COMP_DIR/pnut-sh-base.exe $file $options $(comp_program_options $file) > $COMP_DIR/$output_name-with-$1.sh" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
   elif [ $1 = "pnut" ]; then
-    TIME_MS=$(( `bash -c "time $COMP_DIR/pnut-sh-compiled-by-pnut-exe.exe $file $options > $COMP_DIR/$output_name-with-$1.sh" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
+    TIME_MS=$(( `bash -c "time $COMP_DIR/pnut-sh-compiled-by-pnut-exe.exe $file $options $(comp_program_options $file) > $COMP_DIR/$output_name-with-$1.sh" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
   else
-    TIME_MS=$(( `bash -c "time $1 $COMP_DIR/pnut.sh $file $options > $COMP_DIR/$output_name-with-$1.sh" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
+    TIME_MS=$(( `bash -c "time $1 $COMP_DIR/pnut.sh $file $options $(comp_program_options $file) > $COMP_DIR/$output_name-with-$1.sh" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
   fi
-  print_time $TIME_MS "for: $1 with $file $(sha256sum $COMP_DIR/$output_name-with-$1.sh | cut -d' ' -f1)"
+  print_time $TIME_MS "for: $1 with $file $(sha256sum $COMP_DIR/$output_name-with-$1.sh | cut -d' ' -f1) size: $(wc -l < $file) $(wc -l < $COMP_DIR/$output_name-with-$1.sh)"
 }
 
 PNUT_SH_OPTIONS="-DRT_NO_INIT_GLOBALS -Dsh"
@@ -40,7 +46,7 @@ gcc -o $COMP_DIR/pnut-exe.exe $PNUT_x86_OPTIONS -O3 pnut.c
 
 chmod +x $COMP_DIR/pnut-sh-compiled-by-pnut-exe.exe
 
-programs="examples/empty.c examples/hello.c examples/fib.c examples/cat.c examples/cp.c examples/wc.c examples/sha256sum.c"
+programs="examples/empty.c examples/hello.c examples/fib.c examples/cat.c examples/cp.c examples/wc.c examples/sha256sum.c examples/c4.c examples/repl.c"
 runners="ksh dash bash yash zsh gcc pnut"
 
 for prog in $programs; do

benchmarks/example-execution-times/run.sh

@@ -6,7 +6,7 @@ COMP_DIR="$DIR/compiled"
 # Create the compiled directory if it doesn't exist
 mkdir -p $COMP_DIR
 
-programs="examples/sha256sum.c examples/empty.c examples/hello.c examples/fib.c examples/cat.c examples/cp.c examples/wc.c"
+programs="examples/sha256sum.c examples/empty.c examples/hello.c examples/fib.c examples/cat.c examples/cp.c examples/wc.c examples/c4.c examples/repl.c"
 runners="ksh dash bash yash zsh gcc pnut"
 
 print_time()
@@ -29,21 +29,32 @@ run() {
     chmod +x $executable
     TIME_MS=$(( `bash -c "time ./$executable $options > $COMP_DIR/$name-output-with-$1" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
   else
-    TIME_MS=$(( `bash -c "time $1 $COMP_DIR/$name-with-gcc.sh $options > $COMP_DIR/$name-output-with-$1" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
+    TIME_MS=$(( `bash -c "time $1 $COMP_DIR/$name-on-shell.sh $options > $COMP_DIR/$name-output-with-$1" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
   fi
   print_time $TIME_MS "for: $1 with $name on $options $(sha256sum $COMP_DIR/$name-output-with-$1 | cut -d' ' -f1)"
 }
 
 run_program_options() { # $1 = program, $2 = runner
   case $1 in
     "examples/cat.c")       echo "$COMP_DIR/file-64k" ;;
-    "examples/cp.c")        echo "$COMP_DIR/file-64k $COMP_DIR/file-64k-out-$2" ;;
+    "examples/cp.c")
+      touch "$COMP_DIR/file-64k-out-$2" # gcc and pnut only write to the file if it exists
+      echo "$COMP_DIR/file-64k $COMP_DIR/file-64k-out-$2"
+      ;;
     "examples/wc.c")        echo "$COMP_DIR/file-64k" ;;
     "examples/sha256sum.c") echo "$COMP_DIR/file-64k" ;;
+    "examples/c4.c")        echo "examples/c4.c" ;;
+    "examples/repl.c")      echo "< $COMP_DIR/program.scm" ;;
     *)                      echo "" ;;
   esac
 }
 
+comp_program_options() { # $1 = program
+  case $1 in
+    "examples/c4.c") echo "-I./examples/c4-libs/" ;;
+  esac
+}
+
 # Generate 64k file
 i=0
 rm -f "$COMP_DIR/file-64k"
@@ -52,7 +63,13 @@ while [ $i -lt 128 ] ; do
   : $(( i += 1 ))
 done;
 
-PNUT_SH_OPTIONS="-DRT_NO_INIT_GLOBALS -Dsh"
+# and program.scm
+# (define (fib n) (if (< n 2) n (+ (fib (- n 1)) (fib (- n 2))))) (fib 10)
+# (define (factorial n) (if (= n 0) 1 (* n (factorial (- n 1))))) (factorial 10)
+# (display "Hello, world!")
+echo '(display "Hello, world!")' > $COMP_DIR/program.scm
+
+PNUT_SH_OPTIONS="-DRT_NO_INIT_GLOBALS -Dsh -DOPTIMIZE_LONG_LINES"
 PNUT_x86_OPTIONS="-Dtarget_i386_linux"
 #PNUT_x86_OPTIONS="-Dtarget_x86_64_linux"
 #PNUT_x86_OPTIONS="-Dtarget_x86_64_mac"
@@ -67,15 +84,15 @@ for prog in $programs; do
   # Compile using pnut-sh compiled with gcc and pnut.
   # This makes sure that the programs are compiled correctly.
   name=$(basename $prog ".c")
-  $COMP_DIR/pnut-sh-base.exe $prog > $COMP_DIR/$name-with-gcc.sh
-  ksh $COMP_DIR/pnut.sh $prog > $COMP_DIR/$name-on-shell.sh
+  $COMP_DIR/pnut-sh-base.exe $prog $(comp_program_options $prog) > $COMP_DIR/$name-with-gcc.sh
+  ksh $COMP_DIR/pnut.sh $prog $(comp_program_options $prog) > $COMP_DIR/$name-on-shell.sh
 
-  if ! diff "$COMP_DIR/$name-with-gcc.sh" "$COMP_DIR/$name-on-shell.sh" > /dev/null ; then
-    echo "Warning: Compiling with pnut-sh.exe and pnut-sh.sh produced different outputs"
-    continue
-  fi
+  # if ! diff "$COMP_DIR/$name-with-gcc.sh" "$COMP_DIR/$name-on-shell.sh" > /dev/null ; then
+  #   echo "Warning: Compiling $name with pnut-sh.exe and pnut-sh.sh produced different outputs"
+  #   exit
+  # fi
 
   for runner in $runners; do
-    run "$runner" $prog $(run_program_options $prog $runner)
+    run "$runner" $prog "$(run_program_options $prog $runner)"
   done
 done

examples/repl.c

@@ -5,7 +5,7 @@
  *
  * For more details about ribbit, see the repository: https://github.com/udem-dlteam/ribbit
  *
- * Author : Léonard Oest O'Leary (github.com/leo-ard)
+ * Author : Leonard Oest O'Leary (github.com/leo-ard)
  */
 
 // This repl has been tested against R4RS compliance tests that can be found here :

shell-benchmarks/char-to-int-conversion/run.sh

@@ -6,35 +6,58 @@
 # When the environment is not empty (with 100000 variables initialized), a case statement 2 is faster than a lookup table.
 # In all cases, using printf is at least 4x as slow as the other methods.
 
-DIR="benchmarks/char-to-int-conversion"
+set -e
+
+trap "exit 1" INT
+
+DIR="shell-benchmarks/char-to-int-conversion"
+
+samples=5
 
 print_time()
 {
   ms=$1
   printf "%s %s\n" "$((ms/1000)).$((ms/100%10))$((ms/10%10))$((ms%10))s" "$2"
 }
 
+sample() {
+  S1=$(( `bash -c "time $1" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
+  echo "$S1"
+}
+
+sample_many() { # $1 = number of samples, $2 = command
+  time=0
+  times=""
+  for i in $(seq 1 $1); do
+    s=$(sample "$2")
+    time=$((time + s))
+    times="$times $s"
+  done
+  printf "%s\n" "$times" 1>&2
+  printf "%s" $((time / $1))
+}
+
 with_method() {
   method=$2
 
-  TIME_MS=$(( `bash -c "time $1 $DIR/$method.sh 0 100" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
+  TIME_MS=$(sample_many $samples "$1 $DIR/$method.sh 0 100")
   print_time $TIME_MS "for: $SHELL_TO_TEST $method.sh (empty env)"
 
-  TIME_MS_INIT=$(( `bash -c "time $1 $DIR/$method.sh 1000 0" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
-  TIME_MS=$(( `bash -c "time $1 $DIR/$method.sh 1000 100" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
+  TIME_MS_INIT=$(sample_many $samples "$1 $DIR/$method.sh 1000 0")
+  TIME_MS=$(sample_many $samples "$1 $DIR/$method.sh 1000 100")
   print_time $((TIME_MS - TIME_MS_INIT)) "for: $SHELL_TO_TEST $method.sh (1000 vars)"
 
-  TIME_MS_INIT=$(( `bash -c "time $1 $DIR/$method.sh 10000 0" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
-  TIME_MS=$(( `bash -c "time $1 $DIR/$method.sh 10000 100" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
+  TIME_MS_INIT=$(sample_many $samples "$1 $DIR/$method.sh 10000 0")
+  TIME_MS=$(sample_many $samples "$1 $DIR/$method.sh 10000 100")
   print_time $((TIME_MS - TIME_MS_INIT)) "for: $SHELL_TO_TEST $method.sh (10000 vars)"
 
-  TIME_MS_INIT=$(( `bash -c "time $1 $DIR/$method.sh 100000 0" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
-  TIME_MS=$(( `bash -c "time $1 $DIR/$method.sh 100000 100" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
+  TIME_MS_INIT=$(sample_many $samples "$1 $DIR/$method.sh 100000 0")
+  TIME_MS=$(sample_many $samples "$1 $DIR/$method.sh 100000 100")
   print_time $((TIME_MS - TIME_MS_INIT)) "for: $SHELL_TO_TEST $method.sh (100000 vars)"
 
-  TIME_MS_INIT=$(( `bash -c "time $1 $DIR/$method.sh 500000 0" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
-  TIME_MS=$(( `bash -c "time $1 $DIR/$method.sh 500000 100" 2>&1 | fgrep real | sed -e "s/real[^0-9]*//g" -e "s/m/*60000+/g" -e "s/s//g" -e "s/\\+0\\./-1000+1/g" -e "s/\\.//g"` ))
-  print_time $((TIME_MS - TIME_MS_INIT)) "for: $SHELL_TO_TEST $method.sh (500000 vars)"
+  # TIME_MS_INIT=$(sample_many $samples "$1 $DIR/$method.sh 500000 0")
+  # TIME_MS=$(sample_many $samples "$1 $DIR/$method.sh 500000 100")
+  # print_time $((TIME_MS - TIME_MS_INIT)) "for: $SHELL_TO_TEST $method.sh (500000 vars)"
 }
 
 methods="subshell case lookup_table"