Update GitLab setup and ReBench config

.gitlab-ci.yml

@@ -1,60 +1,63 @@
 stages:
   - build-and-test
-  - benchmark
+  - benchmark-completion
 
 variables:
   PYTHONUNBUFFERED: "true"
 
+before_script:
+  - git submodule update --init
+
 .build_and_test_job: &BUILD_AND_TEST
   stage: build-and-test
   tags: [benchmarks, infinity]
   script:
-    - git submodule update --init
-    - make test
-    - make bench
-    - make clean
-    - make unittests
+    - cmake . $CONF
+    - make
+    - ./SOM++ -cp ./Smalltalk ./TestSuite/TestHarness.som
+    - ./unittests -cp ./Smalltalk:./TestSuite/BasicInterpreterTests ./Examples/Hello.som
+
+    - rebench --experiment="CI ID $CI_PIPELINE_ID" --branch="$CI_COMMIT_REF_NAME" -c rebench.conf all "e:${NAME}"
 
 tagging_gen:
   <<: *BUILD_AND_TEST
-  variables: {USE_TAGGING: "true", GC_TYPE: generational}
+  variables: {CONF: "-DUSE_TAGGING=true -DGC_TYPE=GENERATIONAL", NAME: tag-gen}
 
 tagging_marksweep:
   <<: *BUILD_AND_TEST
-  variables: {USE_TAGGING: "true", GC_TYPE: mark_sweep}
+  variables: {CONF: "-DUSE_TAGGING=true -DGC_TYPE=MARK_SWEEP", NAME: tag-mark}
 
 tagging_copying:
   <<: *BUILD_AND_TEST
-  variables: {USE_TAGGING: "true", GC_TYPE: copying}
+  variables: {CONF: "-DUSE_TAGGING=true -DGC_TYPE=COPYING", NAME: tag-copy}
 
 boxed_generational_cached:
   <<: *BUILD_AND_TEST
-  variables: {USE_TAGGING: "false", GC_TYPE: generational, CACHE_INTEGER: "true"}
+  variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=GENERATIONAL -DCACHE_INTEGER=true", NAME: gen-cache}
 
 boxed_mark_sweep_cached:
   <<: *BUILD_AND_TEST
-  variables: {USE_TAGGING: "false", GC_TYPE: mark_sweep, CACHE_INTEGER: "true"}
+  variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=MARK_SWEEP -DCACHE_INTEGER=true", NAME: mark-cache}
 
 boxed_copying_cached:
   <<: *BUILD_AND_TEST
-  variables: {USE_TAGGING: "false", GC_TYPE: copying, CACHE_INTEGER: "true"}
+  variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=COPYING -DCACHE_INTEGER=true", NAME: copy-cache}
 
 boxed_generational_uncached:
   <<: *BUILD_AND_TEST
-  variables: {USE_TAGGING: "false", GC_TYPE: generational, CACHE_INTEGER: "false"}
+  variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=GENERATIONAL -DCACHE_INTEGER=false", NAME: gen}
 
 boxed_mark_sweep_uncached:
   <<: *BUILD_AND_TEST
-  variables: {USE_TAGGING: "false", GC_TYPE: mark_sweep, CACHE_INTEGER: "false"}
+  variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=MARK_SWEEP -DCACHE_INTEGER=false", NAME: mark}
 
 boxed_copying_uncached:
   <<: *BUILD_AND_TEST
-  variables: {USE_TAGGING: "false", GC_TYPE: copying, CACHE_INTEGER: "false"}
+  variables: {CONF: "-DUSE_TAGGING=false -DGC_TYPE=COPYING -DCACHE_INTEGER=false", NAME: copy}
 
-benchmark_job:
-  stage: benchmark
+
+complete_benchmarking:
+  stage: benchmark-completion
   tags: [benchmarks, infinity]
-  allow_failure: false
   script:
-    - make
-    - rebench -d --without-nice -c --commit-id="$CI_BUILD_REF" --environment="Infinity Ubuntu" --project=SOMpp --branch="$CI_BUILD_REF_NAME" core-lib/codespeed.conf SOMpp
+    - rebench --experiment="CI ID $CI_PIPELINE_ID" --report-completion rebench.conf

rebench.conf

@@ -0,0 +1,84 @@
+# -*- mode: yaml -*-
+# Config file for ReBench
+default_experiment: all
+default_data_file: 'rebench.data'
+
+reporting:
+    # Benchmark results will be reported to ReBenchDB
+    rebenchdb:
+        # this url needs to point to the API endpoint
+        db_url: https://rebench.stefan-marr.de/rebenchdb
+        repo_url: https://github.com/smarr/SOMpp
+        record_all: true # make sure everything is recorded
+        project_name: SOMpp
+
+runs:
+    max_invocation_time: 60
+
+benchmark_suites:
+    macro:
+        gauge_adapter: RebenchLog
+        command: &MACRO_CMD "-cp Smalltalk:Examples/Benchmarks/Richards:Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/NBody:Examples/Benchmarks/Json:Examples/Benchmarks/GraphSearch Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s %(iterations)s 0 "
+        iterations: 10
+        benchmarks:
+            - Richards:     {extra_args: 1}
+            - DeltaBlue:    {extra_args: 50}
+            - NBody:        {extra_args: 500}
+            - JsonSmall:    {extra_args: 1}
+            - GraphSearch:  {extra_args: 4}
+            - PageRank:     {extra_args: 40}
+
+    micro:
+        gauge_adapter: RebenchLog
+        command: "-cp Smalltalk:Examples/Benchmarks/LanguageFeatures Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s %(iterations)s 0 "
+        iterations: 10
+        benchmarks:
+            - Fannkuch:     {extra_args: 6}
+            - Fibonacci:    {extra_args: 3}
+            - Dispatch:     {extra_args: 2}
+            - Bounce:       {extra_args: 2}
+            - Loop:         {extra_args: 5}
+            - Permute:      {extra_args: 3}
+            - Queens:       {extra_args: 2}
+            - List:         {extra_args: 2}
+            - Recurse:      {extra_args: 3}
+            - Storage:      {extra_args: 1}
+            - Sieve:        {extra_args: 4}
+            - BubbleSort:   {extra_args: 3}
+            - QuickSort:    {extra_args: 1}
+            - Sum:          {extra_args: 2}
+            - Towers:       {extra_args: 2}
+            - TreeSort:     {extra_args: 1}
+            - IntegerLoop:  {extra_args: 2}
+            - FieldLoop:    {extra_args: 1}
+            - WhileLoop:    {extra_args: 10}
+            - Mandelbrot:   {extra_args: 30}
+
+executors:
+    tag-gen:    {path: ., executable: SOM++}
+    tag-mark:   {path: ., executable: SOM++}
+    tag-copy:   {path: ., executable: SOM++}
+    gen-cache:  {path: ., executable: SOM++}
+    mark-cache: {path: ., executable: SOM++}
+    copy-cache: {path: ., executable: SOM++}
+    gen:        {path: ., executable: SOM++}
+    mark:       {path: ., executable: SOM++}
+    copy:       {path: ., executable: SOM++}
+
+# define the benchmarks to be executed for a re-executable benchmark run
+experiments:
+    SOM++:
+        description: All benchmarks on CSOM
+        suites:
+            - micro
+            - macro
+        executions:
+            - tag-gen
+            - tag-mark
+            - tag-copy
+            - gen-cache
+            - mark-cache
+            - copy-cache
+            - gen
+            - mark
+            - copy