Merge pull request #120 from cyber-dojo/remove-clean-start-check

Remove clean-start check

bin/build_image.sh

@@ -45,13 +45,22 @@ check_args()
 build_image()
 {
   check_args "$@"
-  local -r type="${1}"
+  local -r type="${1}" # {server|client}
   exit_non_zero_unless_installed docker
-  export $(echo_versioner_env_vars)
+  # shellcheck disable=SC2046
+  export $(echo_env_vars)
 
   containers_down
   remove_old_images
 
+  echo
+  echo "Building with --build-args"
+  echo "  COMMIT_SHA=${COMMIT_SHA}"
+  echo "  BASE_IMAGE=${CYBER_DOJO_SAVER_BASE_IMAGE}"
+  echo "To change this run:"
+  echo "$ COMMIT_SHA=... CYBER_DOJO_SAVER_BASE_IMAGE=cyberdojo/sinatra-base:... make image_${type}"
+  echo
+
   docker compose build server
   if [ "${type}" == 'client' ]; then
     docker compose build client

bin/check_coverage_metrics.sh

@@ -41,7 +41,7 @@ check_args()
 check_coverage()
 {
   check_args "$@"
-  export $(echo_versioner_env_vars)
+  export $(echo_env_vars)
 
   local -r TYPE="${1}"           # {server|client}
   local -r TEST_LOG=test.log

bin/check_test_metrics.sh

@@ -41,7 +41,7 @@ check_args()
 check_coverage()
 {
   check_args "$@"
-  export $(echo_versioner_env_vars)
+  export $(echo_env_vars)
 
   local -r TYPE="${1}"           # {server|client}
   local -r TEST_LOG=test.log

bin/create_almost_full_group.sh

@@ -5,11 +5,14 @@ export ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 
 source "${ROOT_DIR}/bin/lib.sh"
 
-readonly VERSION="${1}"
-export $(echo_versioner_env_vars)
+readonly VERSION="${1}"  # {0|1|2}
+# shellcheck disable=SC2046
+export $(echo_env_vars)
 readonly CONTAINER="${CYBER_DOJO_SAVER_SERVER_CONTAINER_NAME}"
 readonly USER="${CYBER_DOJO_SAVER_SERVER_USER}"
-# TODO: docker command to bring up server
+
+# TODO: Need to start a custom-start-points container
+docker compose --progress=plain up --no-build --wait --wait-timeout=10 server
 docker exec "${CONTAINER}" bash -c "rm -rf /cyber-dojo/*"
 readonly GID=$(docker exec --user "${USER}" "${CONTAINER}" bash -c "ruby /saver/test/data/create_almost_full_group.rb ${VERSION}")
 
@@ -23,7 +26,7 @@ docker exec "${CONTAINER}" \
 
 echo "Filename == ${DST_TGZ_FILENAME}"
 echo
-echo "Now add the following tar_file to run/lib.sh"
+echo "Now add the following tar_file to copy_in_saver_test_data() in run/lib.sh"
 echo
 echo "almost_full_group.v${VERSION}.${GID}.tgz"
 echo

bin/lib.sh

@@ -5,11 +5,18 @@ echo_base_image()
   echo "${json}" | jq -r '.base_image'
 }
 
-echo_versioner_env_vars()
+echo_env_vars()
 {
-  local -r sha="$(cd "${ROOT_DIR}" && git rev-parse HEAD)"
-  echo COMMIT_SHA="${sha}"
+  # --build-arg ...
+  if [[ ! -v CYBER_DOJO_SAVER_BASE_IMAGE ]] ; then
+    echo CYBER_DOJO_SAVER_BASE_IMAGE="$(echo_base_image)"
+  fi
+  if [[ ! -v COMMIT_SHA ]] ; then
+    local -r sha="$(cd "${ROOT_DIR}" && git rev-parse HEAD)"
+    echo COMMIT_SHA="${sha}"
+  fi
 
+  # From versioner ...
   docker run --rm cyberdojo/versioner:latest
 
   echo CYBER_DOJO_SAVER_SHA="${sha}"
@@ -26,10 +33,6 @@ echo_versioner_env_vars()
   local -r AWS_ACCOUNT_ID=244531986313
   local -r AWS_REGION=eu-central-1
   echo CYBER_DOJO_SAVER_IMAGE=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/saver
-
-  if [[ ! -v CYBER_DOJO_SAVER_BASE_IMAGE ]] ; then
-    echo CYBER_DOJO_SAVER_BASE_IMAGE="$(echo_base_image)"
-  fi
 }
 
 stderr()
@@ -88,9 +91,20 @@ copy_in_saver_test_data()
   local -r TEST_DATA_DIR="${ROOT_DIR}/test/server/data"
   local -r CID="${CYBER_DOJO_SAVER_SERVER_CONTAINER_NAME}"
   # You cannot docker cp to a tmpfs, so tar-piping...
-  set -x
-  tar -c -C "${TEST_DATA_DIR}/cyber-dojo" - . | docker exec -i "${CID}" tar x -C /cyber-dojo
-  set +x
+
+  if [ "${CI:-}" == 'true' ]; then
+    set -x
+  fi
+
+  # In the CI workflow this gives a diagnostic
+  #   tar: -: Cannot stat: No such file or directory
+  #   tar: Exiting with failure status due to previous errors
+  # But it appears to worked as the tests all pass, and when commented out some fail ?!?
+  tar --no-xattrs -c -C "${TEST_DATA_DIR}/cyber-dojo" - . | docker exec -i "${CID}" tar x -C /cyber-dojo
+
+  if [ "${CI:-}" == 'true' ]; then
+    set +x
+  fi
 
   local -r tar_files=(
     almost_full_group.v0.AWCQdE.tgz
@@ -135,65 +149,20 @@ remove_all_but_latest()
   docker system prune --force
 }
 
-exit_non_zero_unless_started_cleanly()
+echo_warnings()
 {
+  local -r SERVICE_NAME="${1}"
+  local -r DOCKER_LOG=$(docker logs "${CONTAINER_NAME}" 2>&1)
   # Handle known warnings (eg waiting on Gem upgrade)
   # local -r SHADOW_WARNING="server.rb:(.*): warning: shadowing outer local variable - filename"
   # DOCKER_LOG=$(strip_known_warning "${DOCKER_LOG}" "${SHADOW_WARNING}")
 
-  local -r SERVICE_NAME="${1}"
-  echo
-  echo "Checking if ${SERVICE_NAME} started cleanly."
-
-  local -r DOCKER_LOG=$(docker logs "${CONTAINER_NAME}" 2>&1)
-  local -r top_5=$(echo "${DOCKER_LOG}" | head -5)
-
-  if ! array_prefix "$(clean_top_5)" "${top_5}" ; then
-    echo "${SERVICE_NAME} did not start cleanly: docker log..."
-    echo 'expected------------------'
-    clean_top_5
-    echo
-    echo 'actual--------------------'
-    echo "${top_5}"
-    echo
-    echo 'diff--------------------'
-    grep -Fxvf <(clean_top_5) <(echo "${top_5}")
-    echo
-    exit 42
+  if echo "${DOCKER_LOG}" | grep -q "warning" ; then
+    echo "Warnings in ${SERVICE_NAME} container"
+    echo "${DOCKER_LOG}"
   fi
 }
 
-array_prefix()
-{
-  readarray -t expected_lines <<<"${1}"
-  readarray -t actual_lines <<<"${2}"
-  for i in {0..4}
-  do
-     if ! [[ "${actual_lines[$i]}" =~ ^"${expected_lines[$i]}" ]] ; then
-       return 1 # false
-     fi
-  done
-  return 0  # true
-}
-
-top_5()
-{
-  docker logs "${CONTAINER_NAME}" 2>&1 | head -5
-}
-
-clean_top_5()
-{
-  # 1st 5 lines on Puma
-  local -r L1="Puma starting in single mode..."
-  local -r L2='* Puma version: 6.5.0 ("Sky'"'"'s Version")'
-  local -r L3='* Ruby version: ruby 3.3.6 (2024-11-05 revision 75015d4c1f)'  # [x86_64-linux-musl]
-  local -r L4="*  Min threads: 0"
-  local -r L5="*  Max threads: 5"
-  #
-  local -r all5="$(printf "%s\n%s\n%s\n%s\n%s" "${L1}" "${L2}" "${L3}" "${L4}" "${L5}")"
-  echo "${all5}"
-}
-
 strip_known_warning()
 {
   local -r DOCKER_LOG="${1}"

bin/run_tests.sh

@@ -13,8 +13,8 @@ show_help()
     Use: ${MY_NAME} {server|client} [ID...]
 
     Options:
-       server  - only run tests from inside the server (local only)
-       client  - only run tests from inside the client (local and CI workflow)
+       server  - only run tests from inside the server
+       client  - only run tests from inside the client
        ID...   - only run tests matching these identifiers
 
 EOF
@@ -99,13 +99,14 @@ run_tests_in_container()
 
 run_tests()
 {
-  export $(echo_versioner_env_vars)
+  # shellcheck disable=SC2046
+  export $(echo_env_vars)
   check_args "$@"
   exit_non_zero_unless_installed docker
   containers_down
   create_space_limited_volume
   docker compose --progress=plain up --no-build --wait --wait-timeout=10 "${TYPE}"
-  exit_non_zero_unless_started_cleanly "${TYPE}"
+  echo_warnings "${TYPE}"
   copy_in_saver_test_data
   run_tests_in_container "$@"
 }

bin/snyk_container_scan.sh

@@ -4,7 +4,8 @@ set -Eeu
 export ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 
 source "${ROOT_DIR}/bin/lib.sh"
-export $(echo_versioner_env_vars)
+# shellcheck disable=SC2046
+export $(echo_env_vars)
 
 readonly IMAGE_NAME="${CYBER_DOJO_SAVER_IMAGE}:${CYBER_DOJO_SAVER_TAG}"
 

source/server/external/custom_start_points.rb

@@ -0,0 +1,35 @@
+require_relative 'http_json/service'
+require_relative 'http'
+
+# Used by test/server/data/create_almost_full_group.rb
+
+module External
+
+  class CustomStartPoints
+
+    def initialize
+      service = 'custom-start-points'
+      port = ENV['CYBER_DOJO_CUSTOM_START_POINTS_PORT'].to_i
+      http = External::Http.new
+      @http = HttpJson::service(self.class.name, http, service, port)
+    end
+
+    def ready?
+      @http.get(__method__, {})
+    end
+
+    def display_names
+      @http.get(:names, {})
+    end
+
+    def manifests
+      @http.get(:manifests, {})
+    end
+
+    def manifest(display_name)
+      @http.get(__method__, { name:display_name })
+    end
+
+  end
+
+end

test/server/data/create_almost_full_group.rb

@@ -1,15 +1,12 @@
 # Generates a group with 63 avatars, ready for group-full test
 
 def require_source(file)
-  require_relative("../../source/#{file}")
-end
-def require_test(file)
-  require_relative("../#{file}")
+  require_relative("../../#{file}")
 end
 
 require_source 'externals'
 require_source 'model'
-require_test 'external/custom_start_points'
+require_source 'external/custom_start_points'
 
 version = ARGV[0].to_i
 
@@ -18,9 +15,8 @@ def require_test(file)
 manifest = custom.manifest(name)
 manifest["version"] = version
 model = Model.new(Externals.new)
-manifests = [manifest]
-options = {}
-gid = model.group_create(manifests:manifests, options:options)
+
+gid = model.group_create(manifests:[manifest], options:{})
 63.times { model.group_join(id:gid) }
 
 print(gid)