diff --git a/bin/build_image.sh b/bin/build_image.sh index 34c65aa7..4abe57f2 100755 --- a/bin/build_image.sh +++ b/bin/build_image.sh @@ -45,13 +45,22 @@ check_args() build_image() { check_args "$@" - local -r type="${1}" + local -r type="${1}" # {server|client} exit_non_zero_unless_installed docker - export $(echo_versioner_env_vars) + # shellcheck disable=SC2046 + export $(echo_env_vars) containers_down remove_old_images + echo + echo "Building with --build-args" + echo " COMMIT_SHA=${COMMIT_SHA}" + echo " BASE_IMAGE=${CYBER_DOJO_SAVER_BASE_IMAGE}" + echo "To change this run:" + echo "$ COMMIT_SHA=... CYBER_DOJO_SAVER_BASE_IMAGE=cyberdojo/sinatra-base:... make image_${type}" + echo + docker compose build server if [ "${type}" == 'client' ]; then docker compose build client diff --git a/bin/check_coverage_metrics.sh b/bin/check_coverage_metrics.sh index 98690375..5efc0438 100755 --- a/bin/check_coverage_metrics.sh +++ b/bin/check_coverage_metrics.sh @@ -41,7 +41,7 @@ check_args() check_coverage() { check_args "$@" - export $(echo_versioner_env_vars) + export $(echo_env_vars) local -r TYPE="${1}" # {server|client} local -r TEST_LOG=test.log diff --git a/bin/check_test_metrics.sh b/bin/check_test_metrics.sh index f6fb2fd6..919ff6f4 100755 --- a/bin/check_test_metrics.sh +++ b/bin/check_test_metrics.sh @@ -41,7 +41,7 @@ check_args() check_coverage() { check_args "$@" - export $(echo_versioner_env_vars) + export $(echo_env_vars) local -r TYPE="${1}" # {server|client} local -r TEST_LOG=test.log diff --git a/bin/create_almost_full_group.sh b/bin/create_almost_full_group.sh index 53ac51a8..4def1efd 100755 --- a/bin/create_almost_full_group.sh +++ b/bin/create_almost_full_group.sh @@ -5,11 +5,14 @@ export ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" source "${ROOT_DIR}/bin/lib.sh" -readonly VERSION="${1}" -export $(echo_versioner_env_vars) +readonly VERSION="${1}" # {0|1|2} +# shellcheck disable=SC2046 +export $(echo_env_vars) readonly CONTAINER="${CYBER_DOJO_SAVER_SERVER_CONTAINER_NAME}" readonly USER="${CYBER_DOJO_SAVER_SERVER_USER}" -# TODO: docker command to bring up server + +# TODO: Need to start a custom-start-points container +docker compose --progress=plain up --no-build --wait --wait-timeout=10 server docker exec "${CONTAINER}" bash -c "rm -rf /cyber-dojo/*" readonly GID=$(docker exec --user "${USER}" "${CONTAINER}" bash -c "ruby /saver/test/data/create_almost_full_group.rb ${VERSION}") @@ -23,7 +26,7 @@ docker exec "${CONTAINER}" \ echo "Filename == ${DST_TGZ_FILENAME}" echo -echo "Now add the following tar_file to run/lib.sh" +echo "Now add the following tar_file to copy_in_saver_test_data() in run/lib.sh" echo echo "almost_full_group.v${VERSION}.${GID}.tgz" echo diff --git a/bin/lib.sh b/bin/lib.sh index 09c00d20..4d6fc72e 100644 --- a/bin/lib.sh +++ b/bin/lib.sh @@ -5,11 +5,18 @@ echo_base_image() echo "${json}" | jq -r '.base_image' } -echo_versioner_env_vars() +echo_env_vars() { - local -r sha="$(cd "${ROOT_DIR}" && git rev-parse HEAD)" - echo COMMIT_SHA="${sha}" + # --build-arg ... + if [[ ! -v CYBER_DOJO_SAVER_BASE_IMAGE ]] ; then + echo CYBER_DOJO_SAVER_BASE_IMAGE="$(echo_base_image)" + fi + if [[ ! -v COMMIT_SHA ]] ; then + local -r sha="$(cd "${ROOT_DIR}" && git rev-parse HEAD)" + echo COMMIT_SHA="${sha}" + fi + # From versioner ... docker run --rm cyberdojo/versioner:latest echo CYBER_DOJO_SAVER_SHA="${sha}" @@ -26,10 +33,6 @@ echo_versioner_env_vars() local -r AWS_ACCOUNT_ID=244531986313 local -r AWS_REGION=eu-central-1 echo CYBER_DOJO_SAVER_IMAGE=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/saver - - if [[ ! -v CYBER_DOJO_SAVER_BASE_IMAGE ]] ; then - echo CYBER_DOJO_SAVER_BASE_IMAGE="$(echo_base_image)" - fi } stderr() @@ -88,9 +91,20 @@ copy_in_saver_test_data() local -r TEST_DATA_DIR="${ROOT_DIR}/test/server/data" local -r CID="${CYBER_DOJO_SAVER_SERVER_CONTAINER_NAME}" # You cannot docker cp to a tmpfs, so tar-piping... - set -x - tar -c -C "${TEST_DATA_DIR}/cyber-dojo" - . | docker exec -i "${CID}" tar x -C /cyber-dojo - set +x + + if [ "${CI:-}" == 'true' ]; then + set -x + fi + + # In the CI workflow this gives a diagnostic + # tar: -: Cannot stat: No such file or directory + # tar: Exiting with failure status due to previous errors + # But it appears to worked as the tests all pass, and when commented out some fail ?!? + tar --no-xattrs -c -C "${TEST_DATA_DIR}/cyber-dojo" - . | docker exec -i "${CID}" tar x -C /cyber-dojo + + if [ "${CI:-}" == 'true' ]; then + set +x + fi local -r tar_files=( almost_full_group.v0.AWCQdE.tgz @@ -135,65 +149,20 @@ remove_all_but_latest() docker system prune --force } -exit_non_zero_unless_started_cleanly() +echo_warnings() { + local -r SERVICE_NAME="${1}" + local -r DOCKER_LOG=$(docker logs "${CONTAINER_NAME}" 2>&1) # Handle known warnings (eg waiting on Gem upgrade) # local -r SHADOW_WARNING="server.rb:(.*): warning: shadowing outer local variable - filename" # DOCKER_LOG=$(strip_known_warning "${DOCKER_LOG}" "${SHADOW_WARNING}") - local -r SERVICE_NAME="${1}" - echo - echo "Checking if ${SERVICE_NAME} started cleanly." - - local -r DOCKER_LOG=$(docker logs "${CONTAINER_NAME}" 2>&1) - local -r top_5=$(echo "${DOCKER_LOG}" | head -5) - - if ! array_prefix "$(clean_top_5)" "${top_5}" ; then - echo "${SERVICE_NAME} did not start cleanly: docker log..." - echo 'expected------------------' - clean_top_5 - echo - echo 'actual--------------------' - echo "${top_5}" - echo - echo 'diff--------------------' - grep -Fxvf <(clean_top_5) <(echo "${top_5}") - echo - exit 42 + if echo "${DOCKER_LOG}" | grep -q "warning" ; then + echo "Warnings in ${SERVICE_NAME} container" + echo "${DOCKER_LOG}" fi } -array_prefix() -{ - readarray -t expected_lines <<<"${1}" - readarray -t actual_lines <<<"${2}" - for i in {0..4} - do - if ! [[ "${actual_lines[$i]}" =~ ^"${expected_lines[$i]}" ]] ; then - return 1 # false - fi - done - return 0 # true -} - -top_5() -{ - docker logs "${CONTAINER_NAME}" 2>&1 | head -5 -} - -clean_top_5() -{ - # 1st 5 lines on Puma - local -r L1="Puma starting in single mode..." - local -r L2='* Puma version: 6.5.0 ("Sky'"'"'s Version")' - local -r L3='* Ruby version: ruby 3.3.6 (2024-11-05 revision 75015d4c1f)' # [x86_64-linux-musl] - local -r L4="* Min threads: 0" - local -r L5="* Max threads: 5" - # - local -r all5="$(printf "%s\n%s\n%s\n%s\n%s" "${L1}" "${L2}" "${L3}" "${L4}" "${L5}")" - echo "${all5}" -} - strip_known_warning() { local -r DOCKER_LOG="${1}" diff --git a/bin/run_tests.sh b/bin/run_tests.sh index 3f7d5c3e..9116d7f8 100755 --- a/bin/run_tests.sh +++ b/bin/run_tests.sh @@ -13,8 +13,8 @@ show_help() Use: ${MY_NAME} {server|client} [ID...] Options: - server - only run tests from inside the server (local only) - client - only run tests from inside the client (local and CI workflow) + server - only run tests from inside the server + client - only run tests from inside the client ID... - only run tests matching these identifiers EOF @@ -99,13 +99,14 @@ run_tests_in_container() run_tests() { - export $(echo_versioner_env_vars) + # shellcheck disable=SC2046 + export $(echo_env_vars) check_args "$@" exit_non_zero_unless_installed docker containers_down create_space_limited_volume docker compose --progress=plain up --no-build --wait --wait-timeout=10 "${TYPE}" - exit_non_zero_unless_started_cleanly "${TYPE}" + echo_warnings "${TYPE}" copy_in_saver_test_data run_tests_in_container "$@" } diff --git a/bin/snyk_container_scan.sh b/bin/snyk_container_scan.sh index 371517aa..e8b7c2c3 100755 --- a/bin/snyk_container_scan.sh +++ b/bin/snyk_container_scan.sh @@ -4,7 +4,8 @@ set -Eeu export ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" source "${ROOT_DIR}/bin/lib.sh" -export $(echo_versioner_env_vars) +# shellcheck disable=SC2046 +export $(echo_env_vars) readonly IMAGE_NAME="${CYBER_DOJO_SAVER_IMAGE}:${CYBER_DOJO_SAVER_TAG}" diff --git a/source/server/external/custom_start_points.rb b/source/server/external/custom_start_points.rb new file mode 100644 index 00000000..63848bbf --- /dev/null +++ b/source/server/external/custom_start_points.rb @@ -0,0 +1,35 @@ +require_relative 'http_json/service' +require_relative 'http' + +# Used by test/server/data/create_almost_full_group.rb + +module External + + class CustomStartPoints + + def initialize + service = 'custom-start-points' + port = ENV['CYBER_DOJO_CUSTOM_START_POINTS_PORT'].to_i + http = External::Http.new + @http = HttpJson::service(self.class.name, http, service, port) + end + + def ready? + @http.get(__method__, {}) + end + + def display_names + @http.get(:names, {}) + end + + def manifests + @http.get(:manifests, {}) + end + + def manifest(display_name) + @http.get(__method__, { name:display_name }) + end + + end + +end diff --git a/test/server/data/create_almost_full_group.rb b/test/server/data/create_almost_full_group.rb index f4be7473..21fd2781 100644 --- a/test/server/data/create_almost_full_group.rb +++ b/test/server/data/create_almost_full_group.rb @@ -1,15 +1,12 @@ # Generates a group with 63 avatars, ready for group-full test def require_source(file) - require_relative("../../source/#{file}") -end -def require_test(file) - require_relative("../#{file}") + require_relative("../../#{file}") end require_source 'externals' require_source 'model' -require_test 'external/custom_start_points' +require_source 'external/custom_start_points' version = ARGV[0].to_i @@ -18,9 +15,8 @@ def require_test(file) manifest = custom.manifest(name) manifest["version"] = version model = Model.new(Externals.new) -manifests = [manifest] -options = {} -gid = model.group_create(manifests:manifests, options:options) + +gid = model.group_create(manifests:[manifest], options:{}) 63.times { model.group_join(id:gid) } print(gid)