Pull ccache from S3

.travis.yml

@@ -18,7 +18,7 @@ matrix:
   include:
     - env:
         - VARIANT=obfuscator
-        - ARCH=x86,arm
+        - ARCH=arm,x86
       before_install:
         - pip install --user --upgrade awscli
         - sudo apt-get update
@@ -33,12 +33,13 @@ matrix:
 
     - env:
         - VARIANT=vanilla
-        - ARCH=x86,arm
+        - ARCH=arm,x86
       before_install:
         - pip install --user --upgrade awscli
         - sudo apt-get update
         - sudo apt-get install -y jq
       script:
+        - ./pull_ccache.bash --variant=$VARIANT --arch=$ARCH
         - VARIANT=$VARIANT ./.travis.sh
       after_success:
         - travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" .

Makefile

@@ -6,6 +6,8 @@ SCRIPTS := .travis.sh \
 					 run_build_shell.bash \
 					 stage_sysroot.bash \
 					 do_clang_build.bash \
+					 push_ccache.bash \
+					 pull_ccache.bash \
 
 all: check base build
 
@@ -39,14 +41,24 @@ build-example: check-build_example
 run: check-run_build_shell
 	$(CURDIR)/run_build_shell.bash $(NO_TTY_ARG)
 
-clean-vanilla:
+clean-vanilla-build:
 	docker volume rm vanilla-llvm-build || :
+	docker volume rm vanilla-llvm-ccache || :
+
+clean-vanilla-src:
 	docker volume rm vanilla-llvm || :
 
-clean-obfuscator:
+clean-vanilla: clean-vanilla-build clean-vanilla-src
+
+clean-obfuscator-build:
 	docker volume rm obfuscator-llvm-build || :
+	docker volume rm obfuscator-llvm-ccache || :
+
+clean-obfuscator-src:
 	docker volume rm obfuscator-llvm || :
 
+clean-obfuscator: clean-obfuscator-build clean-obfuscator-src
+
 clean-build: clean-vanilla clean-obfuscator
 
 clean: clean-build

build.bash

@@ -16,7 +16,6 @@ IFS=$'\n\t'
 DOCKER_NAMETAG=$(cat docker_nametag)
 
 mkdir -p output/opt
-mkdir -p output/ccache
 
 VERBOSE=
 NO_TTY=
@@ -27,7 +26,7 @@ while [[ $# -gt 0 ]]; do
   --arch=x86)           ARCH="X86";           shift ;;
   --arch=arm)           ARCH="ARM";           shift ;;
   --arch=arm,x86)       ARCH="ARM\\;X86";     shift ;;
-  --arch=x86,arm)       ARCH="X86\\;ARM";     shift ;;
+  --arch=x86,arm)       ARCH="ARM\\;X86";     shift ;;
 
   --variant=vanilla)    VARIANT="vanilla";    shift ;;
   --variant=obfuscator) VARIANT="obfuscator"; shift ;;
@@ -94,14 +93,12 @@ else
   INTERACTIVE=()
 fi
 
-#    -v $VARIANT-llvm-ccache:/work/ccache \
-
 # shellcheck disable=SC2068
 docker run ${INTERACTIVE[@]:-} --rm \
     -v "$PWD/output/opt:/opt" \
     -v "$PWD/patches:/patches" \
     -v "$PWD:/this_dir" \
-    -v "$PWD/output/ccache:/work/ccache" \
+    -v $VARIANT-llvm-ccache:/work/ccache \
     -v $VARIANT-llvm:/work/$VARIANT-llvm \
     -v $VARIANT-llvm-build:/work/build \
     -e VARIANT=$VARIANT -e ARCH=$ARCH \

describe_repo.bash

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+git describe --tags --always --dirty

publish.sh

@@ -21,7 +21,7 @@ fi
 REPO="${PWD##*/}"
 BUCKET="${BUCKET:-llvm-obfuscator-arm}"
 
-BUILD_VERSION="$(git describe --tags --dirty --always)"
+BUILD_VERSION="$(./describe_repo.bash)"
 BUILD_PATH="$REPO/$BUILD_VERSION"
 if [[ ! -z "$PRODUCT_VERSION" ]]; then
     BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION"
@@ -43,7 +43,6 @@ for file in "$@"; do
     OBJECT="s3://$BUCKET/$KEY"
     if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
         if [[ "$TRAVIS_BRANCH" == master || "$TRAVIS_TAG" == v* || "$TRAVIS_BRANCH" == v*-release ]]; then
-            OBJECT="s3://$BUCKET/$KEY"
             aws s3 cp "$file" "$OBJECT"
         fi
     else

pull_ccache.bash

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2018 Swift Navigation Inc.
+# Contact: Swift Navigation <[email protected]>
+#
+# This source is subject to the license found in the file 'LICENSE' which must
+# be be distributed together with this source. All other rights reserved.
+#
+# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
+# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+
+  --arch=x86)           ARCH="X86";           shift ;;
+  --arch=arm)           ARCH="ARM";           shift ;;
+  --arch=arm,x86)       ARCH="ARM\\;X86";     shift ;;
+  --arch=x86,arm)       ARCH="ARM\\;X86";     shift ;;
+
+  --variant=vanilla)    VARIANT="vanilla";    shift ;;
+  --variant=obfuscator) VARIANT="obfuscator"; shift ;;
+
+  --verbose)            VERBOSE="-v";         shift ;;
+  --no-tty)             NO_TTY=--no-tty;      shift ;;
+
+  *)                                          shift ;;
+  esac
+done
+
+if [[ -z "${ARCH:-}" ]]; then
+  echo "Error: must specify --arch=<arm|x86|arm,x86|x86,arm>"
+  exit 1
+fi
+
+if [[ -z "${VARIANT:-}" ]]; then
+  echo "Error: must a variant to --variant=<vanilla|obfuscator>"
+  exit 1
+fi
+
+BUILD_VERSION="$(./describe_repo.bash)"
+ARCH="$(echo ${ARCH} | sed 's@\\;@-@g')"
+
+CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2"
+
+./s3_download.bash ${CCACHE_ARCHIVE}
+
+docker run ${INTERACTIVE[@]:-} --rm \
+    -v "$PWD:/this_dir" \
+    -v $VARIANT-llvm-ccache:/work/ccache \
+    "$DOCKER_NAMETAG-$VARIANT" \
+    /bin/bash -c "tar -xjf ${CCACHE_ARCHIVE} -C /work/ccache ."

push_ccache.bash

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2018 Swift Navigation Inc.
+# Contact: Swift Navigation <[email protected]>
+#
+# This source is subject to the license found in the file 'LICENSE' which must
+# be be distributed together with this source. All other rights reserved.
+#
+# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
+# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+[[ -z "${DEBUG:-}" ]] || set -x
+
+DOCKER_NAMETAG=$(cat docker_nametag)
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+
+  --arch=x86)           ARCH="X86";           shift ;;
+  --arch=arm)           ARCH="ARM";           shift ;;
+  --arch=arm,x86)       ARCH="ARM\\;X86";     shift ;;
+  --arch=x86,arm)       ARCH="ARM\\;X86";     shift ;;
+
+  --variant=vanilla)    VARIANT="vanilla";    shift ;;
+  --variant=obfuscator) VARIANT="obfuscator"; shift ;;
+
+  --verbose)            VERBOSE="-v";         shift ;;
+  --no-tty)             NO_TTY=--no-tty;      shift ;;
+
+  *)                                          shift ;;
+  esac
+done
+
+if [[ -z "${ARCH:-}" ]]; then
+  echo "Error: must specify --arch=<arm|x86|arm,x86|x86,arm>"
+  exit 1
+fi
+
+if [[ -z "${VARIANT:-}" ]]; then
+  echo "Error: must a variant to build --variant=<vanilla|obfuscator>"
+  exit 1
+fi
+
+BUILD_VERSION="$(./describe_repo.bash)"
+ARCH="$(echo ${ARCH} | sed 's@\\;@-@g')"
+
+CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2"
+
+docker run -it --rm \
+    -v "$PWD:/this_dir" \
+    -v $VARIANT-llvm-ccache:/work/ccache \
+    "$DOCKER_NAMETAG-$VARIANT" \
+    /bin/bash -c "tar -cjf ${CCACHE_ARCHIVE} -C /work/ccache ."
+
+./publish.sh "${CCACHE_ARCHIVE}"

run_build_shell.bash

@@ -37,8 +37,10 @@ docker run -i -t --rm \
     -v "$PWD/bin:/wrapper-bin" \
     -v "$PWD/patches:/patches" \
     -v "$PWD:/this_dir" \
+    -v $VARIANT-llvm-ccache:/work/ccache \
     -v $VARIANT-llvm:/work/$VARIANT-llvm \
     -v $VARIANT-llvm-build:/work/build \
+    -e CCACHE_DIR=/work/ccache \
     "$DOCKER_NAMETAG-$VARIANT" \
     /bin/bash -c "export PATH=/opt/llvm-$VARIANT/bin:/opt/llvm-$VARIANT/wrappers/bin:\$PATH; \
                   cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \

s3_download.bash

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright (C) 2016 Swift Navigation Inc.
+# Contact: Fergus Noble <[email protected]>
+#
+# This source is subject to the license found in the file 'LICENSE' which must
+# be be distributed together with this source. All other rights reserved.
+#
+# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
+# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Script for downloading firmware and NAP binaries from S3 to be incorporated
+# into the Linux image.
+
+set -xe
+
+REPO="${PWD##*/}"
+BUCKET="${BUCKET:-llvm-obfuscator-arm}"
+
+BUILD_VERSION="$(describe_repo.bash)"
+BUILD_PATH="$REPO/$BUILD_VERSION"
+if [[ ! -z "$PRODUCT_VERSION" ]]; then
+    BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION"
+fi
+if [[ ! -z "$PRODUCT_REV" ]]; then
+    BUILD_PATH="$BUILD_PATH/$PRODUCT_REV"
+fi
+if [[ ! -z "$PRODUCT_TYPE" ]]; then
+    BUILD_PATH="$BUILD_PATH/$PRODUCT_TYPE"
+fi
+
+echo "Downloading $* to $BUILD_PATH"
+
+for file in "$@"; do
+    KEY="$BUILD_PATH/$(basename "$file")"
+    OBJECT="s3://$BUCKET/$KEY"
+    aws s3 cp "$OBJECT" "$file" 
+done