Skip to content

Commit

Permalink
ci: use external github action for the namespace report
Browse files Browse the repository at this point in the history
  • Loading branch information
consideRatio committed Jan 11, 2021
1 parent 60b5c5f commit ad32bf0
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 133 deletions.
12 changes: 6 additions & 6 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -147,14 +147,14 @@ jobs:
export BINDER_URL=http://localhost:30901
pytest -m "remote" -v --maxfail=10 --cov binderhub --durations=10 --color=yes
# GitHub Action reference: https://github.com/jupyterhub/action-k8s-namespace-report
- name: Kubernetes namespace report
if: ${{ always() }}
run: |
# Display debugging information and always provide the logs from
# certain important k8s deployments.
. ci/common
full_namespace_report deploy/binder deploy/hub deploy/proxy
uses: jupyterhub/action-k8s-namespace-report@v1
if: always()
with:
important-workloads: deploy/binder deploy/hub deploy/proxy

# GitHub action reference: https://github.com/codecov/codecov-action
- name: Upload coverage stats
uses: codecov/codecov-action@v1
if: ${{ always() }}
128 changes: 1 addition & 127 deletions ci/common
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ await_jupyterhub() {
&& kubectl rollout status --watch --timeout 300s deployment/hub \
&& (
if kubectl get deploy/autohttps > /dev/null 2>&1; then
kubectl rollout status --watch --timeout 300s deployment/autohttps || exit 1
kubectl rollout status --watch --timeout 300s deployment/autohttps
fi
)
}
Expand All @@ -21,129 +21,3 @@ await_binderhub() {
await_jupyterhub
kubectl rollout status --watch --timeout 300s deployment/binder
}

full_namespace_report () {
# This was copied from z2jh 2021-01-06. Work to make it a dedicated GitHub
# action and avoid a duplicated code base is planned. / @consideRatio
# ------------------------------------------------------------------------
#
# Purpose:
# - To chart agnostically print relevant information of the resources in a
# namespace.
#
# Arguments:
# - Accepts a sequence of arguments such as "deploy/hub" "deploy/proxy". It
# will do `kubectl logs --all-containers <arg>` on them.
#
# Relevant references:
# - ContainerStatus ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#containerstatus-v1-core

# printf formatting: a bold bold colored topic with a divider.
yellow=33
red=31
divider='--------------------------------------------------------------------------------'
# GitHub Workflows resets formatting after \n, so its reapplied.
export format="\n\033[${yellow};1m%s\n\033[${yellow};1m${divider}\033[0m\n"
export format_error="\n\033[${red};1m%s\n\033[${red};1m${divider}\033[0m\n"

printf "$format" "# Full namespace report"
printf "$format" "## Resource overview"
# list workloads (deployment,statefulset,daemonset,pod)
printf "$format" "### \$ kubectl get deploy,sts,ds,pod"
kubectl get deploy,sts,ds,pod
# list config (secret,configmap)
printf "$format" "### \$ kubectl get secret,cm"
kubectl get secret,cm
# list networking (service,ingress,networkpolicy)
printf "$format" "### \$ kubectl get svc,ing,netpol"
kubectl get svc,ing,netpol
# list rbac (serviceaccount,role,rolebinding)
printf "$format" "### \$ kubectl get sa,role,rolebinding"
kubectl get sa,role,rolebinding

# Check if any container of any pod has a non ready status
PODS_WITH_NON_READY_CONTAINER=$(
kubectl get pods -o json \
| jq -r '
.items[]
| select(
any(.status.initContainerStatuses[]?; .ready == false)
or
any(.status.containerStatuses[]?; .ready == false)
)
| .metadata.name
'
)
if [ -n "$PODS_WITH_NON_READY_CONTAINER" ]; then
printf "$format_error" "## Pods with non-ready container(s) detected!"
echo "$PODS_WITH_NON_READY_CONTAINER" | xargs --max-args=1 echo -

for var in $PODS_WITH_NON_READY_CONTAINER; do
printf "$format_error" "### \$ kubectl describe pod/$var"
kubectl describe pod/$var
printf "$format_error" "### \$ kubectl logs --all-containers pod/$var"
kubectl logs --all-containers pod/$var || echo # a newline on failure for consistency with non-failure
done

fi

# Check if any container of any pod has a restartCount > 0. Then, we inspect
# their logs with --previous. We also add --follow and --ignore-errors in
# order to ensure we get the information from all containers, and combined
# with --previous it will exit and not get stuck.
#
# ref: https://github.com/kubernetes/kubernetes/issues/97530
PODS_WITH_RESTARTED_CONTAINERS=$(
kubectl get pods -o json \
| jq -r '
.items[]
| select(
any(.status.initContainerStatuses[]?; .restartCount > 0)
or
any(.status.containerStatuses[]?; .restartCount > 0)
)
| .metadata.name
'
)
if [ -n "$PODS_WITH_RESTARTED_CONTAINERS" ]; then
printf "$format_error" "## Pods with restarted containers detected!"
echo "$PODS_WITH_RESTARTED_CONTAINERS" | xargs --max-args=1 echo -

for var in $PODS_WITH_RESTARTED_CONTAINERS; do
printf "$format_error" "### \$ kubectl describe pod/$var"
kubectl describe pod/$var
printf "$format_error" "### \$ kubectl logs --previous --all-containers --follow --ignore-errors pod/$var"
kubectl logs --previous --all-containers --follow --ignore-errors pod/$var
done
fi

# if any pods that should be scheduled by the user-scheduler are pending ->
# show user-scheduler's logs
PENDING_USER_PODS=$(
kubectl get pods -l "component in (user-placeholder,singleuser-server)" -o json \
| jq -r '
.items[]
| select(.status.phase == "Pending")
| .metadata.name
'
)
if [ -n "$PENDING_USER_PODS" ]; then
printf "$format_error" "## Pending pods detected!"
echo "$PENDING_USER_PODS" | xargs --max-args=1 echo -

printf "$format_error" "### \$ kubectl logs --all-containers deploy/user-scheduler"
kubectl logs --all-containers deploy/user-scheduler || echo # a newline on failure for consistency with non-failure
fi

# show container logs of all important workloads passed to the function,
# "deploy/hub" and "deploy/proxy" for example.
if [ "$#" -gt 0 ]; then
printf "$format" "## Important workload's logs"
echo "$@" | xargs --max-args=1 echo -

for var in "$@"; do
printf "$format" "### \$ kubectl logs --all-containers $var"
kubectl logs --all-containers $var || echo # a newline on failure for consistency with non-failure
done
fi
}

0 comments on commit ad32bf0

Please sign in to comment.