kube-down

#!/bin/bash
set -e
source "$(dirname "$0")/runcom" || { echo 'Can not load the rumcom file, exiting...' >&2 && exit 1 ; }

#---

function get_this_container_id(){
  local CONTAINER_ID="$(cat /proc/self/cgroup | sed -n "s|.*\/docker\/\(.*\)|\1|p" | uniq)"
  if [[ "$(echo "${CONTAINER_ID}" | wc -l)" != "1" ]]; then
    echo "'/proc/self/cgroup' may has some problem, exiting..." 1>&2
    exit 1
  fi

  echo "${CONTAINER_ID}"
}

# Return code 0 means this container is k8sup
function check_if_this_container_is_k8sup(){
  local CONTAINER_ID="$(get_this_container_id)"
  if [[ "$(docker inspect "${CONTAINER_ID}" | jq -r '.[0].Name')" == "/k8sup" ]]; then
    return 0
  else
    return 1
  fi
}

function kube_down_save_log(){
  local CONTAINER_ID="$(get_this_container_id)"
  local LOGNAME="kube-down-$(date +"%Y%m%d%H%M%S")"
  mkdir -p "/etc/kubernetes/logs"
  docker logs "${CONTAINER_ID}" &>"/etc/kubernetes/logs/${LOGNAME}.log"
  docker inspect "${CONTAINER_ID}" &>"/etc/kubernetes/logs/${LOGNAME}.json"
}

function remove_node_from_endpoints(){
  local IPPORT_PATTERN="[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}:[0-9]\{1,5\}"
  local IPADDR="$1"
  local HYPERKUBE_IMAGE="$2"
  local APIHOST="$(echo "$3" | grep -o "${IPPORT_PATTERN}")"
  local KUBELET_VARLIB="$4"
  local REMOTE_K8S_IPADDR="$5"
  local APISERVER=""

  if [[ -n "${REMOTE_K8S_IPADDR}" ]]; then
    IPADDR="${REMOTE_K8S_IPADDR}"
  fi

  if [[ "${APIHOST}" == "127.0.0.1"* ]]; then
    APISERVER="http://${APIHOST}"
  elif [[ -n "${APIHOST}" ]]; then
    iptables -w -t nat -I OUTPUT --dest 10.0.0.1 -p tcp --dport 443 -j DNAT --to-dest "${APIHOST}"
    APISERVER="https://10.0.0.1:443"
  else
    return 1
  fi

  local NEW_ENDPOINTS="$(docker run \
    --net=host \
    --rm=true \
    --volume=${KUBELET_VARLIB}/kubeconfig:${KUBELET_VARLIB}/kubeconfig:ro \
    "${HYPERKUBE_IMAGE}" \
    /hyperkube kubectl -s "${APISERVER}" \
      --kubeconfig ${KUBELET_VARLIB}/kubeconfig/kubeconfig.yaml \
      get ep kubernetes --namespace=default -o json \
        | jq ".subsets[].addresses[] | select(.ip != \"${IPADDR}\")" \
        | jq -s -c .)"

  [[ "${NEW_ENDPOINTS}" == "[]" ]] && return 0

  local NEW_PATCH="{\"subsets\":[{\"addresses\":${NEW_ENDPOINTS},\"ports\":[{\"name\":\"https\",\"port\":6443,\"protocol\":\"TCP\"}]}]}"

  docker run \
    --net=host \
    --rm=true \
    --volume=${KUBELET_VARLIB}/kubeconfig:${KUBELET_VARLIB}/kubeconfig:ro \
    "${HYPERKUBE_IMAGE}" \
    /hyperkube kubectl -s "${APISERVER}" \
      --kubeconfig ${KUBELET_VARLIB}/kubeconfig/kubeconfig.yaml \
      patch ep kubernetes \
      --type='merge' \
      -p "${NEW_PATCH}" &>/dev/null \
   && echo "Removed this node from endpoints." 1>&2

  if [[ -n "$(iptables -w -S -t nat | grep 10.0.0.1/32 | grep "${APIHOST}")" ]]; then
    iptables -w -t nat -D OUTPUT --dest 10.0.0.1 -p tcp --dport 443 -j DNAT --to-dest "${APIHOST}"
  fi
}

function get_endpoint_amount(){
  local IPPORT_PATTERN="[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}:[0-9]\{1,5\}"
  local HYPERKUBE_IMAGE="$1"
  local KUBELET_VARLIB="$2"
  local APIHOST="$(echo $3 | grep -o "${IPPORT_PATTERN}")"
  local APISERVER=""

  if [[ "${APIHOST}" == "127.0.0.1"* ]]; then
    APISERVER="http://${APIHOST}"
  elif [[ -n "${APIHOST}" ]]; then
    iptables -w -t nat -I OUTPUT --dest 10.0.0.1 -p tcp --dport 443 -j DNAT --to-dest "${APIHOST}"
    APISERVER="https://10.0.0.1:443"
  else
    return 1
  fi

  local NODE_LIST="$(\
    docker run \
    --net=host \
    --rm=true \
    --volume="${KUBELET_VARLIB}/kubeconfig:${KUBELET_VARLIB}/kubeconfig:ro" \
    "${HYPERKUBE_IMAGE}" \
    /hyperkube kubectl -s "${APISERVER}" \
      --kubeconfig "${KUBELET_VARLIB}/kubeconfig/kubeconfig.yaml" \
      get ep kubernetes --namespace=default -o json \
        | jq -r ".subsets[].addresses[].ip")"
  [[ -z "${NODE_LIST}" ]] && { echo "Could not get API server endpoints!" 1>&2; return 1; }
  [[ "${NODE_LIST}" == "null" ]] && { echo "No such any endpoint!" 1>&2; return 1; }

  if [[ -n "$(iptables -w -S -t nat | grep 10.0.0.1/32 | grep "${APIHOST}")" ]]; then
    iptables -w -t nat -D OUTPUT --dest 10.0.0.1 -p tcp --dport 443 -j DNAT --to-dest "${APIHOST}"
  fi

  local ENDPOINT_AMOUNT="$(echo "${NODE_LIST}" | wc -l)"
  echo "${ENDPOINT_AMOUNT}"
}

# Find an API server
function find_an_apiserver(){
  local ETCD_CLIENT_PORT="$1"
  local K8S_PORT="$2"
  local K8S_INSECURE_PORT="$3"
  local KUBELET_VARLIB="$4"
  local APISERVER
  local NODE
  local IPADDR_PATTERN="[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}"
  local CERTS_DIR="${KUBELET_VARLIB}/kubeconfig"
  local CERT_OPTS="--cert ${CERTS_DIR}/kubecfg.crt \
                   --key ${CERTS_DIR}/kubecfg.key \
                   --cacert ${CERTS_DIR}/ca.crt"

  if curl ${CERT_OPTS} -m 2 -sf "http://127.0.0.1:${K8S_INSECURE_PORT}" &>/dev/null; then
    APISERVER="http://127.0.0.1:${K8S_INSECURE_PORT}"
  else
    # If API server is not running on local, searching it from other nodes.
    local NODE_LIST="$(curl -sf --retry 10 "http://127.0.0.1:${ETCD_CLIENT_PORT}/v2/members" \
                       | jq -r ".members[].clientURLs[0]" \
                       | grep -v 'null' \
                       | grep -o "${IPADDR_PATTERN}")"
    [[ -z "${NODE_LIST}" ]] && { echo "Could not get API server endpoints!" 1>&2; return 1; }

    for NODE in ${NODE_LIST}; do
      if curl -k -m 2 -s "https://${NODE}:${K8S_PORT}" &>/dev/null; then
        APISERVER="https://${NODE}:${K8S_PORT}"
        break
      fi
    done
  fi
  [[ -z "${APISERVER}" ]] && { echo "Could not find any apiserver!" 1>&2; return 1; }

  echo "${APISERVER}"
}

function stop_k8s(){
  local IPPORT_PATTERN="[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}:[0-9]\{1,5\}"
  local IPADDR="$1"
  local ETCD_CLIENT_PORT="$2"
  local KUBELET_VARLIB="$3"
  local HYPERKUBE_IMAGE="$4"
  local APIHOST="$(echo "$5" | grep -o "${IPPORT_PATTERN}")"
  local REMOTE_K8S_IPADDR="$6"
  local EXIT="$7"
  local CONTAINER_IDS=""
  local APISERVER=""

  # Only for exit remote k8s service
  if [[ -n "${REMOTE_K8S_IPADDR}" ]]; then
    IPADDR="${REMOTE_K8S_IPADDR}"
    EXIT="true"
  fi
  if [[ "${APIHOST}" == "127.0.0.1"* ]]; then
    APISERVER="http://${APIHOST}"
  elif [[ -n "${APIHOST}" ]] && [[ -z "${REMOTE_K8S_IPADDR}" ]]; then
    # Only for exit local k8s service
    iptables -w -t nat -I OUTPUT --dest 10.0.0.1 -p tcp --dport 443 -j DNAT --to-dest "${APIHOST}"
    APISERVER="https://10.0.0.1:443"
  else
    return 1
  fi

  # Drain node in preparation for maintenance.
  echo "Evicting service pods..." 1>&2
  local MSG
  until \
      MSG="$(docker run \
        --net=host \
        --rm=true \
        --volume=${KUBELET_VARLIB}/kubeconfig:${KUBELET_VARLIB}/kubeconfig:ro \
        "${HYPERKUBE_IMAGE}" \
        /hyperkube kubectl -s "${APISERVER}" \
          --kubeconfig ${KUBELET_VARLIB}/kubeconfig/kubeconfig.yaml \
          drain "${IPADDR}" --force --ignore-daemonsets --delete-local-data 2>&1)"; do
    [[ "$(echo "${MSG}" | grep "nodes .* not found")" ]] && return 1
    echo "Try again..." 1>&2
    sleep 1
  done
  echo "${MSG}" 1>&2

  if [[ "${EXIT}" == "true" ]]; then
    # Exit k8s cluster
    docker run \
      --net=host \
      --rm=true \
      --volume=${KUBELET_VARLIB}/kubeconfig:${KUBELET_VARLIB}/kubeconfig:ro \
      "${HYPERKUBE_IMAGE}" \
      /hyperkube kubectl -s "${APISERVER}" \
        --kubeconfig ${KUBELET_VARLIB}/kubeconfig/kubeconfig.yaml \
        delete node "${IPADDR}"

    # Stop all OSD containers on this node
    CONTAINER_IDS="$(docker ps | grep -E "k8s_ceph-osd-pod.[[:alnum:]]{7,8}_ceph-osd-[[:alnum:]]{5}_ceph" | awk '{print $1}')"
    [[ -n "${CONTAINER_IDS}" ]] && docker exec ${CONTAINER_IDS} ceph-api stop_all_osds 1>/dev/null

    # Waiting for all non-kube-system pods leave
    until ! \
        docker run \
          --net=host \
          --rm=true \
          --volume=${KUBELET_VARLIB}/kubeconfig:${KUBELET_VARLIB}/kubeconfig:ro \
          "${HYPERKUBE_IMAGE}" \
          /hyperkube kubectl -s "${APISERVER}" \
            --kubeconfig ${KUBELET_VARLIB}/kubeconfig/kubeconfig.yaml \
            get po --all-namespaces -o wide \
              | grep -v "\<kube-system\>" \
              | grep "\<${IPADDR}\>" &>/dev/null; do
      sleep 2
    done
  fi

  # Only for exit local k8s service
  if [[ -z "${REMOTE_K8S_IPADDR}" ]]; then
    # Remove k8s system pods conf
    echo '{}' | tee /etc/kubernetes/manifests-multi/master-multi.json &>/dev/null
    echo '{}' | tee /etc/kubernetes/manifests-multi/addon-manager-multinode.json &>/dev/null

    # Stop kubelet, kube-addon-manager
    docker stop k8sup-kubelet 1>/dev/null || true
    CONTAINER_IDS="$(docker ps | grep -E "k8s_.*(k8s-proxy).*_kube-system" | awk '{print $1}')"
    [[ -n "${CONTAINER_IDS}" ]] && docker stop ${CONTAINER_IDS} 1>/dev/null || true
    CONTAINER_IDS="$(docker ps | grep -E "k8s_.*(kube-addon-manager-|setup).*_kube-system" | awk '{print $1}')"
    [[ -n "${CONTAINER_IDS}" ]] && docker kill ${CONTAINER_IDS} 1>/dev/null || true
    CONTAINER_IDS="$(docker ps | grep -E "k8s_.*(k8s-master).*_kube-system" | awk '{print $1}')"
    [[ -n "${CONTAINER_IDS}" ]] && docker stop ${CONTAINER_IDS} 1>/dev/null || true
    [[ -n "${CONTAINER_IDS}" ]] && docker rm -v ${CONTAINER_IDS} 1>/dev/null || true
    # Stop other k8s component services
    CONTAINER_IDS="$(docker ps | grep -E "k8s_([[:alnum:]]|-)+\.[[:alnum:]]{7,8}_.*_kube-system" | awk '{print $1}')"
    [[ -n "${CONTAINER_IDS}" ]] && docker stop ${CONTAINER_IDS} 1>/dev/null || true
    CONTAINER_IDS="$(docker ps -a | grep -E "k8s_([[:alnum:]]|-)+\.[[:alnum:]]{7,8}_.*_kube-system" | awk '{print $1}')"
    [[ -n "${CONTAINER_IDS}" ]] && docker rm -v ${CONTAINER_IDS} 1>/dev/null || true

    echo -n "Waiting for all k8s pods stopped..." 1>&2
    until [[ "$(docker ps | grep "${HYPERKUBE_IMAGE}" | wc -l)" -le "2" ]]; do
      echo -n "." 1>&2
      sleep 1
    done
    echo 1>&2

    until ! docker ps | grep -w 'k8sup-kubelet'; do
      docker stop k8sup-kubelet 1>/dev/null || true
      docker rm k8sup-kubelet 1>/dev/null || true
    done

    if [[ -n "$(iptables -w -S -t nat | grep 10.0.0.1/32 | grep "${APIHOST}")" ]]; then
      iptables -w -t nat -D OUTPUT --dest 10.0.0.1 -p tcp --dport 443 -j DNAT --to-dest "${APIHOST}"
    fi
  else
    echo "K8S node ${IPADDR} has been exited." 1>&2
  fi
}

function exit_etcd(){
  local IPADDR="$1"
  local ETCD_CLIENT_PORT="$2"
  local REMOTE_ETCD_IPADDR="$3"

  [[ -n "${REMOTE_ETCD_IPADDR}" ]] && IPADDR="${REMOTE_ETCD_IPADDR}"

  # Exit etcd cluster
  local MEMBER_LIST="$(curl -s http://127.0.0.1:${ETCD_CLIENT_PORT}/v2/members)"
  if [[ "${MEMBER_LIST}" == *"${IPADDR}:${ETCD_CLIENT_PORT}"* ]]; then
    local MEMBER_ID="$(echo "${MEMBER_LIST}" | jq -r ".members[] | select(contains({clientURLs: [\"/${IPADDR}:\"]})) | .id")"
    test "${MEMBER_ID}" && curl -s "http://127.0.0.1:${ETCD_CLIENT_PORT}/v2/members/${MEMBER_ID}" -XDELETE
  fi
  if [[ -n "${REMOTE_ETCD_IPADDR}" ]]; then
    echo "etcd node ${IPADDR} has been exited." 1>&2
  else
    docker stop k8sup-etcd 1>/dev/null
    docker rm k8sup-etcd 1>/dev/null
    rm -rf "/var/lib/etcd/"*
  fi
}

function show_usage(){
  USAGE="Usage: ${0##*/} [options...]
Options:
-r, --remove                             Exit K8S cluster and remove data
    --stop-k8s-only                      Just stop k8s service without stop etcd service
-f, --force                              Force to kill named k8sup containers
    --exit-local-etcd-only               Exit the local etcd member from cluster
    --exit-remote-etcd=REMOTE_IPADDR     Exit the remote etcd member from cluster
    --exit-remote-k8s=REMOTE_IPADDR      Exit the remote k8s node from cluster
    --debug                              Enable debug mode
-h, --help                               This help text
"

  echo "${USAGE}"
}

function get_options(){
  local PROGNAME="${0##*/}"
  local SHORTOPTS="rfh"
  local LONGOPTS="remove,stop-k8s-only,force,exit-local-etcd-only,exit-remote-etcd:,exit-remote-k8s:,debug,help"
  local PARSED_OPTIONS=""
  local IPADDR_PATTERN="[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}"

  PARSED_OPTIONS="$(getopt -o "${SHORTOPTS}" --long "${LONGOPTS}" -n "${PROGNAME}" -- "$@")" || exit 1
  eval set -- "${PARSED_OPTIONS}"

  # extract options and their arguments into variables.
  while true ; do
      case "$1" in
          -r|--remove)
              export EX_EXIT="true"
              shift
              ;;
             --stop-k8s-only)
              export EX_STOP_K8S_ONLY="true"
              shift
              ;;
          -f|--force)
              export EX_FORCE="true"
              shift
              ;;
             --exit-local-etcd-only)
              export EX_EXIT_LOCAL_ETCD_ONLY="true"
              shift
              ;;
             --exit-remote-etcd)
              export EX_REMOTE_ETCD_IPADDR="$2"
              shift 2
              ;;
             --exit-remote-k8s)
              export EX_REMOTE_K8S_IPADDR="$2"
              shift 2
              ;;
             --debug)
              set -x
              export SHELLOPTS
              shift
              ;;
          -h|--help)
              show_usage
              exit 0
              shift
              ;;
          --)
              shift
              break
              ;;
          *)
              echo "Option error!" 1>&2
              echo $1
              exit 1
              ;;
      esac
  done

  if [[ -n "${EX_REMOTE_ETCD_IPADDR}" ]] \
   && [[ -z "$(echo "${EX_REMOTE_ETCD_IPADDR}" | grep -o "${IPADDR_PATTERN}")" ]]; then
    echo "Need IP address of remote etcd node, exiting..." 1>&2
    exit 1
  fi
  if [[ -n "${EX_REMOTE_K8S_IPADDR}" ]] \
   && [[ -z "$(echo "${EX_REMOTE_K8S_IPADDR}" | grep -o "${IPADDR_PATTERN}")" ]]; then
    echo "Need IP address of remote k8s node, exiting..." 1>&2
    exit 1
  fi
}

function main(){
  get_options "$@"
  source "/root/.bashrc"

  if check_if_this_container_is_k8sup; then
    echo "Container named k8sup can not run kube-down by itself, please run another container to do this." 1>&2
    exit 1
  fi

  local IPADDR="${EX_IPADDR}"
  local ETCD_CLIENT_PORT="${EX_ETCD_CLIENT_PORT}"
  local K8S_VERSION="${EX_K8S_VERSION}"
  local K8S_PORT="${EX_K8S_PORT}"
  local K8S_INSECURE_PORT="${EX_K8S_INSECURE_PORT}"
  local REGISTRY="${EX_REGISTRY}"
  local HYPERKUBE_IMAGE="${EX_HYPERKUBE_IMAGE}"

  local EXIT="${EX_EXIT}"
  local STOP_K8S_ONLY="${EX_STOP_K8S_ONLY}"
  local EXIT_LOCAL_ETCD_ONLY="${EX_EXIT_LOCAL_ETCD_ONLY}"
  local START_ETCD_ONLY="${EX_START_ETCD_ONLY}"
  local REMOTE_ETCD_IPADDR="${EX_REMOTE_ETCD_IPADDR}"
  local REMOTE_K8S_IPADDR="${EX_REMOTE_K8S_IPADDR}"
  local FORCE="${EX_FORCE}"
  local KUBELET_VARLIB="/var/lib/kubelet"
  local APISERVER
  local ENDPOINT_AMOUNT

  if [[ "${FORCE}" != "true" ]]; then
    echo "Check the etcd status..." 1<&2
    local ETCD_WRITE_TEST_URL="http://127.0.0.1:${ETCD_CLIENT_PORT}/v2/keys/k8sup/cluster/write_test"
    if ! curl -sf -m 3 "${ETCD_WRITE_TEST_URL}" -XPUT -d value="${IPADDR}" &>/dev/null; then
      echo "etcd is unhealthy (not writable), exiting..." 1<&2
      exit 1
    else
      curl -sf -m 3 "${ETCD_WRITE_TEST_URL}" -XDELETE &>/dev/null
    fi
  fi

  if [[ "${REMOTE_ETCD_IPADDR}" == "${IPADDR}" ]] \
   || [[ "${REMOTE_K8S_IPADDR}" == "${IPADDR}" ]];then
    echo "Remote IP address can not be local IP address, exiting..." 1>&2
    exit 1
  fi

  if [[ "${EXIT_LOCAL_ETCD_ONLY}" == "true" ]]; then
    exit_etcd "${IPADDR}" "${ETCD_CLIENT_PORT}" "${REMOTE_ETCD_IPADDR}"
    exit 0
  fi
  if [[ -n "${REMOTE_ETCD_IPADDR}" ]]; then
    exit_etcd "${REMOTE_IPADDR}" "${ETCD_CLIENT_PORT}" "${REMOTE_ETCD_IPADDR}"
    exit 0
  fi
  if [[ "${FORCE}" == "true" ]]; then
    echo "Trying to kill k8s containers..." 1>&2
    CONTAINER_IDS="$(docker ps -a | grep -E "k8sup-kubelet" | awk '{print $1}')"
    [[ -n "${CONTAINER_IDS}" ]] && docker stop ${CONTAINER_IDS} 1>/dev/null || true
    CONTAINER_IDS="$(docker ps -a \
      | grep "k8s_.*_[[:alnum:]]" \
      | awk '{print $1}')"
    [[ -n "${CONTAINER_IDS}" ]] && docker rm -f -v ${CONTAINER_IDS} 1>/dev/null || true
    if [[ "${EXIT}" == "true" ]]; then
      exit_etcd "${IPADDR}" "${ETCD_CLIENT_PORT}" "${REMOTE_ETCD_IPADDR}"
    fi
  else
    # Default removing
    if [[ "${START_ETCD_ONLY}" != "true" ]]; then
      echo "Stopping k8s containers........." 1>&2
      APISERVER="$(find_an_apiserver "${ETCD_CLIENT_PORT}" "${K8S_PORT}" "${K8S_INSECURE_PORT}" "${KUBELET_VARLIB}")" || exit 1
      stop_k8s "${IPADDR}" "${ETCD_CLIENT_PORT}" "${KUBELET_VARLIB}" "${HYPERKUBE_IMAGE}" "${APISERVER}" "${REMOTE_K8S_IPADDR}" "${EXIT}" || true
      cleanup_old_kubeproxy_iptables "${HYPERKUBE_IMAGE}"
      APISERVER="$(find_an_apiserver "${ETCD_CLIENT_PORT}" "${K8S_PORT}" "${K8S_INSECURE_PORT}" "${KUBELET_VARLIB}")" \
        && ENDPOINT_AMOUNT="$(get_endpoint_amount "${HYPERKUBE_IMAGE}" "${KUBELET_VARLIB}" "${APISERVER}")" \
        || ENDPOINT_AMOUNT="0"
      if [[ "${ENDPOINT_AMOUNT}" -gt "1" ]]; then
        # Try to find another apiserver for removing this node from endpoints
        APISERVER="$(find_an_apiserver "${ETCD_CLIENT_PORT}" "${K8S_PORT}" "${K8S_INSECURE_PORT}" "${KUBELET_VARLIB}")" \
         && remove_node_from_endpoints "${IPADDR}" "${HYPERKUBE_IMAGE}" "${APISERVER}" "${KUBELET_VARLIB}" "${REMOTE_K8S_IPADDR}" || true
      fi
    fi
    if [[ "${STOP_K8S_ONLY}" == "true" ]] || [[ -n "${REMOTE_K8S_IPADDR}" ]]; then
      exit 0
    fi
    if [[ "${EXIT}" == "true" ]]; then
      exit_etcd "${IPADDR}" "${ETCD_CLIENT_PORT}" "${REMOTE_ETCD_IPADDR}"
    fi
  fi

  echo "Stopping k8sup-kubelet, k8sup-etcd, k8sup-flanneld, and k8sup..." 1>&2
  CONTAINER_IDS="$(docker ps -a | grep -E "k8s-proxy|k8sup-kubelet|k8sup-flannel|k8sup-etcd|k8sup-certs" | awk '{print $1}')"
  [[ -n "${CONTAINER_IDS}" ]] && docker stop ${CONTAINER_IDS} 1>/dev/null || true
  [[ -n "${CONTAINER_IDS}" ]] && docker rm -v ${CONTAINER_IDS} 1>/dev/null || true

  docker stop k8sup 1>/dev/null || true
  docker rm -v k8sup 1>/dev/null || true

  kube_down_save_log

  echo "Done" 1>&2
}

main "$@"