Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add CI validation of shell scripts using shellcheck #4826

Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .github/workflows/ci-validation-of-shell-scripts.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: Validation Of Shell Scripts

on:
push:
branches: [main]

pull_request:
branches: [main]

jobs:
validation-of-shell-scripts:
runs-on: ubuntu-latest

steps:
- name: check out code
uses: actions/checkout@v3

- name: Install shellcheck
run: sudo apt-get install shellcheck

- name: Run shellcheck
run: shellcheck scripts/*.sh






8 changes: 4 additions & 4 deletions scripts/build-all-in-one-image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ make build-ui

run_integration_test() {
local image_name="$1"
CID=$(docker run -d -p 16686:16686 -p 5778:5778 ${image_name}:${GITHUB_SHA})
CID=$(docker run -d -p 16686:16686 -p 5778:5778 "${image_name}:${GITHUB_SHA}")
if ! make all-in-one-integration-test ; then
echo "---- integration test failed unexpectedly ----"
echo "--- check the docker log below for details ---"
docker logs $CID
docker logs "$CID"
exit 1
fi
docker kill $CID
docker kill "$CID"
}

if [ "$mode" = "pr-only" ]; then
Expand All @@ -58,7 +58,7 @@ bash scripts/build-upload-a-docker-image.sh -b -c all-in-one -d cmd/all-in-one -

# build debug image if not on a pull request
if [ "$mode" != "pr-only" ]; then
make build-all-in-one-debug GOOS=linux GOARCH=$GOARCH
make build-all-in-one-debug GOOS=linux GOARCH="$GOARCH"
repo=${repo}-debug

# build all-in-one-debug image locally for integration test
Expand Down
2 changes: 1 addition & 1 deletion scripts/build-crossdock.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ if [[ "$BRANCH" == "main" ]]; then
docker buildx build --push \
--progress=plain \
--platform=linux/amd64 \
${IMAGE_TAGS} \
"${IMAGE_TAGS[@]}" \
yurishkuro marked this conversation as resolved.
Show resolved Hide resolved
crossdock/
else
echo 'skip docker images upload for PR'
Expand Down
3 changes: 2 additions & 1 deletion scripts/build-upload-a-docker-image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ else
fi
fi

docker buildx build --output "${PUSHTAG}" \
# shellcheck disable=SC2086
docker buildx build --output ${PUSHTAG} \
akagami-harsh marked this conversation as resolved.
Show resolved Hide resolved
--progress=plain ${target_arg} ${base_debug_img_arg}\
--platform=${platforms} \
--file ${docker_file_arg} \
Expand Down
16 changes: 10 additions & 6 deletions scripts/cassandra-integration-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,16 @@ setup_cassandra() {
--publish 9042:9042
--publish 9160:9160
)
local cid=$(docker run ${params[@]} ${image}:${tag})
echo ${cid}
local cid
yurishkuro marked this conversation as resolved.
Show resolved Hide resolved
# shellcheck disable=SC2068
cid=$(docker run ${params[@]} "${image}:${tag}")
yurishkuro marked this conversation as resolved.
Show resolved Hide resolved
echo "${cid}"
}

teardown_cassandra() {
local cid=$1
docker kill ${cid}
exit ${exit_status}
docker kill "${cid}"
exit "${exit_status}"
}

apply_schema() {
Expand All @@ -45,17 +47,19 @@ apply_schema() {
--network host
)
docker build -t ${image} ${schema_dir}
# shellcheck disable=SC2068
docker run ${params[@]} ${image}
akagami-harsh marked this conversation as resolved.
Show resolved Hide resolved
}

run_integration_test() {
local version=$1
local schema_version=$2
local cid=$(setup_cassandra ${version})
local cid
cid=$(setup_cassandra "${version}")
apply_schema "$2"
STORAGE=cassandra make storage-integration-test
exit_status=$?
trap "teardown_cassandra ${cid}" EXIT
trap "teardown_cassandra \${cid}" EXIT
yurishkuro marked this conversation as resolved.
Show resolved Hide resolved
}

main() {
Expand Down
29 changes: 17 additions & 12 deletions scripts/check-go-version.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,17 @@ while getopts "uvd" opt; do
case $opt in
u) update=true ;;
v) verbose=true ;;
x) set -x ;;
*) echo "Usage: $0 [-u] [-v] [-d]" >&2
exit 1
;;
esac
done

# Debugging option (-x) used separately
akagami-harsh marked this conversation as resolved.
Show resolved Hide resolved
if [[ "$DEBUG" == "true" ]]; then
set -x
fi

# Fetch latest go release version
go_latest_version=$(curl -s https://go.dev/dl/?mode=json | jq -r '.[0].version' | awk -F'.' '{gsub("go", ""); print $1"."$2}')
go_previous_version="${go_latest_version%.*}.$((10#${go_latest_version#*.} - 1))"
Expand All @@ -31,27 +35,27 @@ function update() {
old_IFS=$IFS
IFS=''
while read -r line; do
match=$(echo $line | grep -e "$pattern")
match=$(echo "$line" | grep -e "$pattern")
if [[ "$match" != "" ]]; then
line=$(echo "$line" | sed "s/${current}/${target}/g")
line=${line//${current}/${target}}
fi
echo $line >> $newfile
done < $file
echo "$line" >> "$newfile"
done < "$file"
IFS=$old_IFS

if [ $verbose = true ]; then
diff $file $newfile
diff "$file" "$newfile"
fi

mv $newfile $file
mv "$newfile" "$file"
}

function check() {
local file=$1
local pattern=$2
local target=$3

go_version=$(grep -e "$pattern" $file | head -1 | sed "s/^.*\($version_regex\).*$/\1/")
go_version=$(grep -e "$pattern" "$file" | head -1 | sed "s/^.*\($version_regex\).*$/\1/")

if [ "$go_version" = "$target" ]; then
mismatch=''
Expand All @@ -68,16 +72,17 @@ function check() {
printf "%-50s Go version: %s %s\n" "$file" "$go_version" "$mismatch"
}

check go.mod "^go\s\+$version_regex" $go_previous_version
check go.mod "^go\s\+$version_regex" "$go_previous_version"

check docker/Makefile "^.*golang:$version_regex" $go_latest_version
check docker/Makefile "^.*golang:$version_regex" "$go_latest_version"

gha_workflows=$(grep -rl go-version .github)
# shellcheck disable=SC2068
for gha_workflow in ${gha_workflows[@]}; do
akagami-harsh marked this conversation as resolved.
Show resolved Hide resolved
check $gha_workflow "^\s*go-version:\s\+$version_regex" $go_latest_version
check "$gha_workflow" "^\s*go-version:\s\+$version_regex" "$go_latest_version"
done

check .golangci.yml "go:\s\+\"$version_regex\"" $go_previous_version
check .golangci.yml "go:\s\+\"$version_regex\"" "$go_previous_version"

if [ $files_to_update -eq 0 ]; then
echo "All files are up to date."
Expand Down
10 changes: 5 additions & 5 deletions scripts/check-test-files.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ COLOR_FIXME=$(printf "\033[31mFIXME\033[0m")
NO_TEST_FILE_DIRS=""
# shellcheck disable=SC2048
for dir in $*; do
mainFile=$(find ${dir} -maxdepth 1 -name 'main.go')
testFiles=$(find ${dir} -maxdepth 1 -name '*_test.go')
mainFile=$(find "${dir}" -maxdepth 1 -name 'main.go')
testFiles=$(find "${dir}" -maxdepth 1 -name '*_test.go')
if [ -z "${testFiles}" ]; then
if [ -n "${mainFile}" ]; then
continue # single main does not require tests
fi
if [ -e ${dir}/.nocover ]; then
reason=$(cat ${dir}/.nocover)
if [ -e "${dir}"/.nocover ]; then
reason=$(cat "${dir}"/.nocover)
akagami-harsh marked this conversation as resolved.
Show resolved Hide resolved
if [ "${reason}" == "" ]; then
echo "error: ${dir}/.nocover must specify reason" >&2
exit 1
Expand All @@ -33,7 +33,7 @@ done

if [ -n "${NO_TEST_FILE_DIRS}" ]; then
echo "*** directories without *_test.go files:" >&2
echo ${NO_TEST_FILE_DIRS} | tr ' ' '\n' >&2
echo "${NO_TEST_FILE_DIRS}" | tr ' ' '\n' >&2
echo "error: at least one *_test.go file must be in all directories with go files so that they are counted for code coverage" >&2
echo " if no tests are possible for a package (e.g. it only defines types), create empty_test.go" >&2
exit 1
Expand Down
2 changes: 2 additions & 0 deletions scripts/compute-tags.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ BRANCH=${BRANCH:?'expecting BRANCH env var'}

## if we are on a release tag, let's extract the version number
## the other possible value, currently, is 'main' (or another branch name)
# shellcheck disable=SC2086
yurishkuro marked this conversation as resolved.
Show resolved Hide resolved
if [[ $BRANCH =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
MAJOR_MINOR_PATCH=$(echo ${BRANCH} | grep -Po "([\d\.]+)")
MAJOR_MINOR=$(echo ${MAJOR_MINOR_PATCH} | awk -F. '{print $1"."$2}')
Expand Down Expand Up @@ -36,4 +37,5 @@ fi

IMAGE_TAGS="${IMAGE_TAGS} --tag docker.io/${SNAPSHOT_TAG} --tag quay.io/${SNAPSHOT_TAG}"

#shellcheck disable=SC2086
akagami-harsh marked this conversation as resolved.
Show resolved Hide resolved
echo ${IMAGE_TAGS}
42 changes: 24 additions & 18 deletions scripts/es-integration-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@ setup_es() {
--env "xpack.security.enabled=false"
--env "xpack.monitoring.enabled=false"
)
local cid=$(docker run ${params[@]} ${image}:${tag})
echo ${cid}
local cid
# shellcheck disable=SC2068
cid=$(docker run ${params[@]} "${image}:${tag}")
echo "${cid}"
}

setup_opensearch() {
Expand All @@ -43,8 +45,10 @@ setup_opensearch() {
--env "transport.host=127.0.0.1"
--env "plugins.security.disabled=true"
)
local cid=$(docker run ${params[@]} ${image}:${tag})
echo ${cid}
local cid
# shellcheck disable=SC2068
cid=$(docker run ${params[@]} "${image}:${tag}")
akagami-harsh marked this conversation as resolved.
Show resolved Hide resolved
echo "${cid}"
}

wait_for_storage() {
Expand All @@ -56,21 +60,23 @@ wait_for_storage() {
--output
/dev/null
--write-out
''%{http_code}''
"%{http_code}"
)
local counter=0
local max_counter=60
while [[ "$(curl ${params[@]} ${url})" != "200" && ${counter} -le ${max_counter} ]]; do
docker inspect ${cid} | jq '.[].State'
# shellcheck disable=SC2068
while [[ "$(curl ${params[@]} "${url}")" != "200" && ${counter} -le ${max_counter} ]]; do
docker inspect "${cid}" | jq '.[].State'
echo "waiting for ${url} to be up..."
sleep 10
counter=$((counter+1))
done
# after the loop, do final verification and set status as global var
if [[ "$(curl ${params[@]} ${url})" != "200" ]]; then
# shellcheck disable=SC2068
if [[ "$(curl ${params[@]} "${url}")" != "200" ]]; then
echo "ERROR: ${distro} is not ready"
docker logs ${cid}
docker kill ${cid}
docker logs "${cid}"
docker kill "${cid}"
db_is_up=0
else
echo "SUCCESS: ${distro} is ready"
Expand All @@ -87,21 +93,21 @@ bring_up_storage() {
for retry in 1 2 3
do
echo "attempt $retry"
if [ ${distro} = "elasticsearch" ]; then
cid=$(setup_es ${version})
elif [ ${distro} == "opensearch" ]; then
cid=$(setup_opensearch ${version})
if [ "${distro}" = "elasticsearch" ]; then
cid=$(setup_es "${version}")
elif [ "${distro}" == "opensearch" ]; then
cid=$(setup_opensearch "${version}")
else
echo "Unknown distribution $distro. Valid options are opensearch or elasticsearch"
usage
fi
wait_for_storage ${distro} "http://localhost:9200" ${cid}
wait_for_storage "${distro}" "http://localhost:9200" "${cid}"
if [ ${db_is_up} = "1" ]; then
break
fi
done
if [ ${db_is_up} = "1" ]; then
trap "teardown_storage ${cid}" EXIT
trap 'teardown_storage ${cid}' EXIT
yurishkuro marked this conversation as resolved.
Show resolved Hide resolved
else
echo "ERROR: unable to start ${distro}"
exit 1
Expand All @@ -110,15 +116,15 @@ bring_up_storage() {

teardown_storage() {
local cid=$1
docker kill ${cid}
docker kill "${cid}"
}

main() {
check_arg "$@"
local distro=$1
local version=$2

bring_up_storage ${distro} ${version}
bring_up_storage "${distro}" "${version}"
STORAGE=${distro} make storage-integration-test
make index-cleaner-integration-test
make index-rollover-integration-test
Expand Down
2 changes: 1 addition & 1 deletion scripts/generate-help-output.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ function gen {
shift
for s in "$@"
do
SPAN_STORAGE_TYPE=$s go run ./cmd/$bin help > $dir/$bin-$s.txt
SPAN_STORAGE_TYPE=$s go run ./cmd/"$bin" help > "$dir"/"$bin"-"$s".txt
akagami-harsh marked this conversation as resolved.
Show resolved Hide resolved
done
}

Expand Down
7 changes: 4 additions & 3 deletions scripts/hotrod-integration-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@ make prepare-docker-buildx
#build image locally for integration test
bash scripts/build-upload-a-docker-image.sh -l -c example-hotrod -d examples/hotrod -p "${platforms}"

export CID=$(docker run -d -p 8080:8080 localhost:5000/$REPO:${GITHUB_SHA})
export CID
CID=$(docker run -d -p 8080:8080 localhost:5000/$REPO:"${GITHUB_SHA}")
yurishkuro marked this conversation as resolved.
Show resolved Hide resolved
i=0
while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8080)" != "200" && ${i} -lt 30 ]]; do
while [[ "$(curl -s -o /dev/null -w "%{http_code}" localhost:8080)" != "200" && ${i} -lt 30 ]]; do
akagami-harsh marked this conversation as resolved.
Show resolved Hide resolved
sleep 1
i=$((i+1))
done
Expand All @@ -24,6 +25,6 @@ if [[ $body != *"Rides On Demand"* ]]; then
echo "String \"Rides On Demand\" is not present on the index page"
exit 1
fi
docker rm -f $CID
docker rm -f "$CID"

bash scripts/build-upload-a-docker-image.sh -c example-hotrod -d examples/hotrod -p "${platforms}"
4 changes: 2 additions & 2 deletions scripts/import-order-cleanup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

set -e

# shellcheck disable=SC2046 -- we want multple arguments here
./scripts/import-order-cleanup.py -o $1 -t $(git ls-files "*\.go" | \
# shellcheck disable=SC2046 # we want multple arguments here
./scripts/import-order-cleanup.py -o "$1" -t $(git ls-files "*\.go" | \
grep -v \
-e thrift-gen \
-e swagger-gen \
Expand Down
Loading