Simplify generating and gathering test coverage metrics

.github/workflows/main.yml

@@ -260,7 +260,7 @@ jobs:
         with:
           version: ${{ vars.KOSLI_CLI_VERSION }}
 
-      - name: Attest junit test evidence to Kosli
+      - name: Attest JUnit test evidence to Kosli
         if: ${{ github.ref == 'refs/heads/main' && (success() || failure()) }}
         env:
           KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }}
@@ -278,7 +278,7 @@ jobs:
           kosli attest generic \
               --description="unit-test branch-coverage and metrics" \
               --name=runner.unit-test-branch-coverage \
-              --user-data="./reports/server/summary.json"
+              --user-data="./reports/server/coverage_metrics.json"
 
 
   integration-tests:
@@ -332,7 +332,7 @@ jobs:
           kosli attest generic \
               --description="integration-test branch-coverage and metrics" \
               --name=runner.integration-test-branch-coverage \
-              --user-data="./reports/client/summary.json"
+              --user-data="./reports/client/coverage_metrics.json"
 
 
   snyk-container-scan:

Dockerfile

@@ -1,4 +1,4 @@
-FROM cyberdojo/docker-base:9f558cc
+FROM cyberdojo/docker-base:b3a6744
 LABEL [email protected]
 
 RUN gem install --no-document 'concurrent-ruby'

bin/build_image.sh

@@ -72,7 +72,7 @@ build_image()
   if [ "${type}" == 'server' ]; then
     # Create latest tag for image build cache
     docker tag "${image_name}" "${CYBER_DOJO_RUNNER_IMAGE}:latest"
-    # Tag image-name for local development where differs name comes from echo-versioner-env-vars
+    # Tag image-name for local development where runners name comes from echo-versioner-env-vars
     docker tag "${image_name}" "cyberdojo/runner:latest"
     echo "CYBER_DOJO_RUNNER_SHA=${CYBER_DOJO_RUNNER_SHA}"
     echo "CYBER_DOJO_RUNNER_TAG=${CYBER_DOJO_RUNNER_TAG}"

bin/check_coverage.sh

@@ -42,31 +42,28 @@ check_coverage()
 {
   check_args "$@"
   local -r TYPE="${1}"           # {server|client}
-  local -r TEST_LOG=test.run.log
+  local -r TEST_LOG=test.log
   local -r HOST_TEST_DIR="${ROOT_DIR}/test/${TYPE}"
-  local -r HOST_REPORTS_DIR="${ROOT_DIR}/reports/${TYPE}"  # where report files have been written to
-  local -r CONTAINER_TMP_DIR=/tmp                          # where to mount to in container
+  local -r HOST_REPORTS_DIR="${ROOT_DIR}/reports/${TYPE}"  # where report json files have been written to
+  local -r CONTAINER_TMP_DIR=/tmp
 
-  exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/${TEST_LOG}"
-  exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/index.html"
-  exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/summary.json"
-  exit_non_zero_unless_file_exists "${HOST_TEST_DIR}/max_metrics.json"
+  exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/test_metrics.json"
+  exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/coverage_metrics.json"
 
   set +e
   docker run \
     --rm \
-    --env CODE_DIR=app \
-    --env TEST_DIR=test \
-    --volume ${HOST_REPORTS_DIR}/${TEST_LOG}:${CONTAINER_TMP_DIR}/${TEST_LOG}:ro \
-    --volume ${HOST_REPORTS_DIR}/summary.json:${CONTAINER_TMP_DIR}/summary.json:ro \
-    --volume ${HOST_TEST_DIR}/max_metrics.json:${CONTAINER_TMP_DIR}/max_metrics.json:ro \
-    cyberdojo/check-test-metrics:latest \
-      "${CONTAINER_TMP_DIR}/${TEST_LOG}" \
-      "${CONTAINER_TMP_DIR}/summary.json" \
-      "${CONTAINER_TMP_DIR}/max_metrics.json" \
-    | tee -a "${HOST_REPORTS_DIR}/${TEST_LOG}"
+    --entrypoint="" \
+    --env COVERAGE_ROOT="${CONTAINER_TMP_DIR}" \
+    --env COVERAGE_CODE_TAB_NAME=app \
+    --env COVERAGE_TEST_TAB_NAME=test \
+    --volume ${HOST_REPORTS_DIR}/test_metrics.json:${CONTAINER_TMP_DIR}/test_metrics.json:ro \
+    --volume ${HOST_REPORTS_DIR}/coverage_metrics.json:${CONTAINER_TMP_DIR}/coverage_metrics.json:ro \
+    --volume ${HOST_TEST_DIR}/check_test_metrics.rb:${CONTAINER_TMP_DIR}/check_test_metrics.rb:ro \
+      cyberdojo/runner:latest \
+        sh -c "ruby ${CONTAINER_TMP_DIR}/check_test_metrics.rb"
 
-  local -r STATUS=${PIPESTATUS[0]}
+  local -r STATUS=$?
   set -e
 
   echo "${TYPE} coverage status == ${STATUS}"

bin/lib.sh

@@ -55,16 +55,16 @@ echo_versioner_env_vars()
 
   echo CYBER_DOJO_RUNNER_SHA="${sha}"
   echo CYBER_DOJO_RUNNER_TAG="${sha:0:7}"
-  #
+
   echo CYBER_DOJO_RUNNER_CLIENT_IMAGE=cyberdojo/runner-client
   echo CYBER_DOJO_RUNNER_CLIENT_PORT=9999
-  #
+
   echo CYBER_DOJO_RUNNER_CLIENT_USER=nobody
   echo CYBER_DOJO_RUNNER_SERVER_USER=root
-  #
+
   echo CYBER_DOJO_RUNNER_CLIENT_CONTAINER_NAME=test_runner_client
   echo CYBER_DOJO_RUNNER_SERVER_CONTAINER_NAME=test_runner_server
-  #
+
   local -r AWS_ACCOUNT_ID=244531986313
   local -r AWS_REGION=eu-central-1
   echo CYBER_DOJO_RUNNER_IMAGE="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/runner"
@@ -100,9 +100,7 @@ exit_non_zero_unless_started_cleanly()
   local -r SERVICE_NAME="${1}"
   echo
   echo "Checking if ${SERVICE_NAME} started cleanly."
-  if [ "$(top_5)" == "$(clean_top_5)" ]; then
-    echo "${SERVICE_NAME} started cleanly."
-  else
+  if [ "$(top_5)" != "$(clean_top_5)" ]; then
     echo "${SERVICE_NAME} did not start cleanly: docker log..."
     echo 'expected------------------'
     echo "$(clean_top_5)"

bin/run_tests.sh

@@ -89,7 +89,8 @@ run_tests_in_container()
   # Check we generated the expected files.
   exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/${TEST_LOG}"
   exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/index.html"
-  exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/summary.json"
+  exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/coverage_metrics.json"
+  exit_non_zero_unless_file_exists "${HOST_REPORTS_DIR}/test_metrics.json"
 
   echo "${TYPE} test branch-coverage report is at:"
   echo "${HOST_REPORTS_DIR}/index.html"

source/client/Dockerfile

@@ -1,4 +1,4 @@
-FROM cyberdojo/docker-base:9f558cc
+FROM cyberdojo/docker-base:b3a6744
 LABEL [email protected]
 
 WORKDIR /runner

test/client/check_test_metrics.rb

@@ -0,0 +1,61 @@
+
+# Uses data from two json files:
+# - reports/client/test_metrics.json     generated in slim_json_reporter.rb by minitest. See id58_test_base.rb
+# - reports/client/coverage_metrics.json generated in simplecov_formatter_json.rb by simplecov. See coverage.rb
+
+require 'json'
+
+def coloured(tf)
+  red = 31
+  green = 32
+  colourize(tf ? green : red, tf)
+end
+
+def colourize(code, word)
+  "\e[#{code}m #{word} \e[0m"
+end
+
+def table_data
+  cov_root = ENV['COVERAGE_ROOT']
+  stats = JSON.parse(IO.read("#{cov_root}/test_metrics.json"))
+
+  cov_json = JSON.parse(IO.read("#{cov_root}/coverage_metrics.json"))
+  test_cov = cov_json['groups'][ENV['COVERAGE_TEST_TAB_NAME']]
+  code_cov = cov_json['groups'][ENV['COVERAGE_CODE_TAB_NAME']]
+
+  [
+    [ nil ],
+    [ 'test.count',    stats['test_count'],    '>=',  67 ],
+    [ 'test.duration', stats['total_time'],    '<=', 100 ],
+    [ nil ],
+    [ 'test.failures', stats['failure_count'], '<=',  0 ],
+    [ 'test.errors',   stats['error_count'  ], '<=',  0 ],
+    [ 'test.skips',    stats['skip_count'   ], '<=',  0 ],
+    [ nil ],
+    [ 'test.lines.total',      test_cov['lines'   ]['total' ], '<=', 518 ],
+    [ 'test.lines.missed',     test_cov['lines'   ]['missed'], '<=', 0   ],
+    [ 'test.branches.total',   test_cov['branches']['total' ], '<=', 5   ],
+    [ 'test.branches.missed',  test_cov['branches']['missed'], '<=', 0   ],
+    [ nil ],
+    [ 'code.lines.total',      code_cov['lines'   ]['total' ], '<=', 122 ],
+    [ 'code.lines.missed',     code_cov['lines'   ]['missed'], '<=', 1   ],
+    [ 'code.branches.total',   code_cov['branches']['total' ], '<=', 6   ],
+    [ 'code.branches.missed',  code_cov['branches']['missed'], '<=', 0   ],
+  ]
+end
+
+results = []
+table_data.each do |name, value, op, limit|
+  if name.nil?
+    puts
+    next
+  end
+  # puts "name=#{name}, value=#{value}, op=#{op}, limit=#{limit}"  # debug
+  result = eval("#{value} #{op} #{limit}")
+  puts "%s | %s %s %s | %s" % [
+    name.rjust(25), value.to_s.rjust(5), "  #{op}", limit.to_s.rjust(5), coloured(result)
+  ]
+  results << result
+end
+puts
+exit results.all?

test/id58_test_base.rb

@@ -1,10 +1,16 @@
 # frozen_string_literal: true
 require 'English'
 require 'minitest/autorun'
-require 'minitest/ci'
+require 'minitest/reporters'
 require_relative 'require_code'
+require_relative 'slim_json_reporter'
 
-Minitest::Ci.report_dir = "#{ENV.fetch('COVERAGE_ROOT', nil)}/junit"
+reporters = [
+  Minitest::Reporters::DefaultReporter.new,
+  Minitest::Reporters::SlimJsonReporter.new,
+  Minitest::Reporters::JUnitReporter.new("#{ENV.fetch('COVERAGE_ROOT')}/junit")
+]
+Minitest::Reporters.use!(reporters)
 
 class Id58TestBase < Minitest::Test
   def initialize(arg)
@@ -52,7 +58,7 @@ def self.define_test(os, display_name, id58_suffix, *lines, &test_block)
   Minitest.after_run do
     slow = @@timings.select { |_name, secs| secs > 0.000 }
     sorted = slow.sort_by { |_name, secs| -secs }.to_h
-    size = [sorted.size, 25].min
+    size = [sorted.size, 5].min
     puts
     puts "Slowest #{size} tests are..." if size != 0
     sorted.each_with_index do |(name, secs), index|

test/lib/coverage.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 require 'simplecov'
-require_relative 'simplecov_json'
+require_relative 'simplecov_formatter_json'
 
 CONTEXT = ENV.fetch('CONTEXT', nil)
 

test/lib/simplecov_json.rb → test/lib/simplecov_formatter_json.rb

@@ -43,7 +43,7 @@ def output_filepath
       end
 
       def output_filename
-        'summary.json'
+        'coverage_metrics.json'
       end
 
       def output_message(result)

test/server/check_test_metrics.rb

@@ -0,0 +1,61 @@
+
+# Uses data from two json files:
+# - reports/server/test_metrics.json     generated in slim_json_reporter.rb by minitest. See id58_test_base.rb
+# - reports/server/coverage_metrics.json generated in simplecov_formatter_json.rb by simplecov. See coverage.rb
+
+require 'json'
+
+def coloured(tf)
+  red = 31
+  green = 32
+  colourize(tf ? green : red, tf)
+end
+
+def colourize(code, word)
+  "\e[#{code}m #{word} \e[0m"
+end
+
+def table_data
+  cov_root = ENV['COVERAGE_ROOT']
+  stats = JSON.parse(IO.read("#{cov_root}/test_metrics.json"))
+
+  cov_json = JSON.parse(IO.read("#{cov_root}/coverage_metrics.json"))
+  test_cov = cov_json['groups'][ENV['COVERAGE_TEST_TAB_NAME']]
+  code_cov = cov_json['groups'][ENV['COVERAGE_CODE_TAB_NAME']]
+
+  [
+    [ nil ],
+    [ 'test.count',    stats['test_count'],    '>=', 99 ],
+    [ 'test.duration', stats['total_time'],    '<=', 10 ],
+    [ nil ],
+    [ 'test.failures', stats['failure_count'], '<=',  0 ],
+    [ 'test.errors',   stats['error_count'  ], '<=',  0 ],
+    [ 'test.skips',    stats['skip_count'   ], '<=',  0 ],
+    [ nil ],
+    [ 'test.lines.total',      test_cov['lines'   ]['total' ], '<=', 967 ],
+    [ 'test.lines.missed',     test_cov['lines'   ]['missed'], '<=', 0   ],
+    [ 'test.branches.total',   test_cov['branches']['total' ], '<=', 0   ],
+    [ 'test.branches.missed',  test_cov['branches']['missed'], '<=', 0   ],
+    [ nil ],
+    [ 'code.lines.total',      code_cov['lines'   ]['total' ], '<=', 549 ],
+    [ 'code.lines.missed',     code_cov['lines'   ]['missed'], '<=', 1   ],
+    [ 'code.branches.total',   code_cov['branches']['total' ], '<=', 66  ],
+    [ 'code.branches.missed',  code_cov['branches']['missed'], '<=', 2   ],
+  ]
+end
+
+results = []
+table_data.each do |name, value, op, limit|
+  if name.nil?
+    puts
+    next
+  end
+  # puts "name=#{name}, value=#{value}, op=#{op}, limit=#{limit}"  # debug
+  result = eval("#{value} #{op} #{limit}")
+  puts "%s | %s %s %s | %s" % [
+    name.rjust(25), value.to_s.rjust(5), "  #{op}", limit.to_s.rjust(5), coloured(result)
+  ]
+  results << result
+end
+puts
+exit results.all?

test/slim_json_reporter.rb

@@ -0,0 +1,20 @@
+require 'json'
+require 'minitest/reporters'
+
+class Minitest::Reporters::SlimJsonReporter < Minitest::Reporters::BaseReporter
+
+  def report
+    super
+    filename = "#{ENV.fetch('COVERAGE_ROOT')}/test_metrics.json"
+    metrics = {
+      "total_time": "%.2f" % total_time,
+      "assertion_count": assertions,
+      "test_count": count,
+      "failure_count": failures,
+      "error_count": errors,
+      "skip_count": skips
+    }
+    File.write(filename, JSON.pretty_generate(metrics))
+  end
+
+end