[ci] Updates ci based on the vault changes (#77)

ci/pipeline/jobs/prepare.yml

@@ -5,9 +5,9 @@ jobs:
   plan:
   - do:
     - in_parallel:
-      - { get: version,    passed: [deploy, upgrade], params: {bump: final} }
-      - { get: spec-check, passed: [deploy, upgrade] }
-      - { get: git,        passed: [deploy, upgrade], trigger: true }
+      - { get: version,    passed: [deploy,upgrade], params: {bump: final} }
+      - { get: spec-check, passed: [deploy,upgrade] }
+      - { get: git,        passed: [deploy,upgrade], trigger: true }
       - { get: git-ci }
       - { get: git-latest-tag }
       - { get: release-notes }

ci/pipeline/jobs/upgrade.yml

@@ -29,7 +29,7 @@ jobs:
         KIT_SHORTNAME:        (( grab meta.kit ))
         SKIP_FRESH:           true
         SKIP_REPLACE_SECRETS: true
-        SKIP_SMOKE_TESTS:     true
+        SKIP_SMOKE_TESTS:     false
         SKIP_CLEAN:           false
     on_failure:
       put: notify

ci/repipe

@@ -16,7 +16,7 @@ need_command() {
   local cmd=${1:?need_command() - no command name given}
   local url=${2:-}
 
-  if [[ ! -x "$(type -p "$cmd")" ]]; then
+  if [[ ! -x "$(type -P "$cmd")" ]]; then
     echo >&2 "${cmd} is not installed."
     if [[ -n "$url" ]]; then
       echo >&2 "Please download it from ${url}"
@@ -96,7 +96,7 @@ OPTIONS:
              whatever is set in 'meta.exposed' in the settings.yml file)
   -o         Open pipeline in browser if os supports it (mac only currently)
              after applying changes. Specify twice to not do anything else.
-  --fly <x>  Path to fly command, otherwise will use $(type -p fly)
+  --fly <x>  Path to fly command, otherwise will use $(type -P fly)
 
 EOF
   exit $rc
@@ -167,7 +167,7 @@ need_command jq
 
 if [[ -z "$fly" ]] ; then
   need_command fly;
-  fly="$(type -p fly)"
+  fly="$(type -P fly)"
 fi
 
 # -- Get settings file --------------------------------------------------------
@@ -220,7 +220,7 @@ if (( DRYRUN > 0 )) ; then
     if [[ -n "${persistent_file}" ]] ; then
       if [[ "${persistent_file}" =~ '->0x' ]] ; then
         persistent_file=''
-      elif [[ -n "$(type -p realpath || true)" ]] ; then
+      elif [[ -n "$(type -P realpath || true)" ]] ; then
         persistent_file="$(realpath --relative-to="$call_dir" "$persistent_file")"
       fi
     fi

ci/scripts/build-kit

@@ -33,15 +33,15 @@ echo "$VAULT_TOKEN" | safe auth token
 safe read secret/handshake
 
 check_dirs=()
-for dir in overlay manifests; do
+for dir in overlay manifests spec/results; do
   [[ -d "$REPO_ROOT/$dir" ]] && check_dirs+=( "$REPO_ROOT/$dir/" )
 done
 if [[ ${#check_dirs[@]} -gt 0 ]] ; then
   header "Checking SHA1s of specified components (not including bosh-deployment) ..."
   out="$(eval "spruce merge --skip-eval $( \
     grep -rl '^releases:' "${check_dirs[@]}" \
-    | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \
-    ) | spruce json | jq -r ." )"
+    | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: [ \"(( merge on sha1 ))\", .releases[] ]}')/" |tr "\n" " " \
+  ) | spruce json | jq -r ." )"
   echo "$out" | spruce merge | spruce json | "${CI_ROOT}/ci/scripts/check-sha1s"
 fi
 

ci/scripts/build-upstream-jobs

@@ -14,15 +14,29 @@ upstream_details="$(spruce json "${base_dir}/settings.yml" | jq -r '.meta.upstre
 # For each release in upstream.yml,
 for release in $upstream_details ; do
   name="$(_lookup "$release" .name)"
-  type="$(_lookup "$release" '.type//"bosh-io-release"')" # Other valid value is github-release
+  type="$(_lookup "$release" '.type//"bosh-io-release"')"
   path="$(_lookup "$release" '.path//"manifests/releases/'"$name"'.yml"')"
   repo="$(_lookup "$release" '.repository')"
-  owner=""
-  if [[ $type == "github-release" ]] ; then
-    owner=$'\n'"      owner: ${repo%/*}"
-    repo="${repo##*/}"
-  fi
+  if [[ $type == 'bosh-io-release' ]] ; then
+    source=$'\n'"      repository: $repo";
+  elif [[ $type == 'github-release' ]] ; then
+    owner="$(_lookup "$release" '.owner//""')"
+    if [[ -z "$owner" && "$repo" =~ / ]] ; then
+      owner="${repo%%/*}"
+      repo="${repo#*/}"
+    fi
+    source=$'\n'"      repository: $repo"$'\n'"      owner: $owner";
 
+    token="$(_lookup "$release" '.access_token//""')"
+    if [[ -n "$token" ]] ; then
+      source="$source"$'\n      access_token: "'"$token"'"'
+    fi
+  else
+    echo >&2 "Unknown resource type for $name upstream release: $type"
+    echo >&2 "Expecting one of: bosh-io-release, github-release"
+    echo >&2 "Update upstream.bosh-releases configuration in ci/settings.yml"
+    exit 1
+  fi
   job="update-${name}-release"
   release="${name}-release"
 
@@ -63,18 +77,18 @@ resources:
   - name: $release
     type: $type
     check_every: 24h
-    source:
-      repository: $repo$owner
+    source: $source
 EOF
 
 done
 group_file="$base_dir/pipeline/upstream/update_group.yml"
 if [[ "${#update_group[@]}" -gt 0 ]] ; then
   (
   echo "groups:"
-  echo "- (( append ))"
+  echo "- (( merge on name ))"
   echo "- name: upstream"
   echo "  jobs:"
+  echo "  - (( append ))"
   for job in ${update_group[@]+"${update_group[@]}"} ; do
     echo "  - $job"
   done

ci/scripts/compare-release-specs

@@ -3,7 +3,7 @@ set -ue
 
 # What branch is the comparison of the current working branch being compared against
 compare_branch="${1:-origin/}"
-check_dirs="spec manifests manifests/releases"
+check_dirs="spec/results manifests"
 
 orig_dir="$(pwd)"
 # needed because of when running locally or in ci
@@ -23,24 +23,28 @@ release_files() {
 releases() {
   eval "spruce merge --skip-eval $( \
   release_files \
-  | sed -e "s/\\(.*\\)/\<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \
+  | sed -e "s/\\(.*\\)/\<(spruce json \\1 | jq -r '{releases: [ \"(( merge on sha1 ))\", .releases[] ]}')/" |tr "\n" " " \
   ) | spruce merge | spruce json | jq -r ."
 }
 
 workdir="$(mktemp -d)"
 mkdir "$workdir/compare"
 cp -R "$(pwd)/.git" "$workdir/compare/"
 pushd "$workdir/compare" > /dev/null
+if ! git show -q "$compare_branch" &>/dev/null ; then
+  echo "The latest release ($compare_branch) is NOT an ancestor to this commit."
+  echo "This should never happen -- Cannot continue!"
+ exit 1
+fi
 git checkout -qf --detach "$compare_branch"
 prev_releases="$(releases)"
 popd > /dev/null
 rm -rf "$workdir/compare"
 
 curr_releases="$(releases)"
 
-
-prev_rel_names="$(echo "$prev_releases"| jq -r '.releases[] | .name' | sort)"
-curr_rel_names="$(echo "$curr_releases"| jq -r '.releases[] | .name' | sort)"
+prev_rel_names="$(echo "$prev_releases"| jq -r '.releases[] | .name' | sort | uniq)"
+curr_rel_names="$(echo "$curr_releases"| jq -r '.releases[] | .name' | sort | uniq)"
 
 removed=()
 while IFS='' read -r rel ; do
@@ -54,9 +58,11 @@ done <<<"$(diff -p <(echo "$prev_rel_names") <(echo "$curr_rel_names") | grep '^
 unchanged=()
 changed=()
 while IFS='' read -r rel; do
-  prev_ver="$(echo "$prev_releases" | jq -r --arg r "$rel" '(.releases[] | select(.name == $r) | .version ) // "--none--" ' )"
+  prev_ver="$(echo "$prev_releases" | jq -r --arg r "$rel" \
+    '.releases | map(select(.name == $r) | .version) | sort | unique | if(.|length>0) then .|join(",") else "--none--" end' )"
   if [[ "$prev_ver" == "--none--" ]] ; then continue ; fi
-  curr_ver="$(echo "$curr_releases" | jq -r --arg r "$rel" '.releases[] | select(.name == $r) | .version' )"
+  curr_ver="$(echo "$curr_releases" | jq -r --arg r "$rel" \
+    '.releases | map(select(.name == $r) | .version) | sort | unique | join(",")' )"
   if [[ "$prev_ver" == "$curr_ver" ]] ; then
     unchanged+=( "$rel $curr_ver" )
   else
@@ -102,28 +108,38 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then
   if [ -f "${ci_dir}/ci/upstreamrepo.yml" ]; then
     upstreamrepo=$(spruce json "${ci_dir}/ci/upstreamrepo.yml")
   else
-    upstreamrepo="[]"
+    upstreamrepo='{"repos": []}'
   fi
+  # TODO: do this in two phases -- first phase pull out all the non-compiled
+  #       versions, then run through with the compiled versions, picking up the
+  #       non-compiled version's git repo.  Also indicate if they are compiled
+  #       or not, and if so, what os is the target. (because that may change)
   repos="$(
     echo "$curr_releases" \
     | jq --argjson gitrepos "$upstreamrepo" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []};
         ($url
-        | if ($url | test("https?://s3.amazonaws.com")) then
+        | if ($url | test("https?://s3(-.*)?.amazonaws.com")) then
             ($gitrepos.repos | map(select(.name == $name))[0].repo)
-            elif ($url | test("https?://bosh.io")) then
+          elif ($url | test("https?://storage.googleapis.com")) then
+            ($gitrepos.repos | map(select(.name == $name))[0].repo)
+          elif ($url | test("https?://bosh.io")) then
             ($url | sub("^.*/d/";"https://") | sub("\\?v=.*$";""))
-            elif ($url | test("https?://github.com")) then
+          elif ($url | test("https?://github.com")) then
             ($url | sub("^.*http";"http") | sub("/releases/download/.*$";""))
-            else
+          else
             $url
-            end
+          end
         ) as $repo |
         (.repos += [{$name,$repo}])
     )')"
 
   for info in "${changed[@]}" ; do
     read -r rel prev_ver curr_ver <<<"$info"
-    repo="$(echo "$repos" | jq -r --arg r "$rel" '.repos[] | select(.name == $r) | .repo' )"
+
+    #TODO: handle multiple versions (comma separated) -- right now we're just taking the first one with a repo.
+    #TODO: handle compiled releases better -- right now just skipping.
+    repo="$(echo "$repos" | jq -r --arg r "$rel" '.repos | map(select(.name == $r and .repo != null)) | .[0].repo//""' )"
+    [[ -n "$repo" ]] || continue
     rel_dir="$workdir/releases/$rel"
     mkdir -p "$rel_dir"
     git -C "$rel_dir" init >/dev/null 2>&1 && \

ci/scripts/test-addons

@@ -0,0 +1,4 @@
+genesis "do" "${DEPLOY_ENV}" -- download-fly
+genesis "do" "${DEPLOY_ENV}" -- login
+genesis "do" "${DEPLOY_ENV}" -- fly teams -d
+genesis "do" "${DEPLOY_ENV}" -- logout