sync_full

#! /usr/bin/env bash

set -euo pipefail

PATH="$(dirname "$(readlink -e "$0")"):$PATH"


function fetch_list_of_remote_files() {
    local -r s3_url=$1
    s3cmd -r ls "$s3_url" |\
        sed -r -e 's|[0-9-]+[[:space:]]+[0-9:]+[[:space:]]+[0-9]+[[:space:]]+'"$s3_url"'/?([^/].+)|\1|' |\
        sort
}

function list_all_local_files() {
    local -r source_dir=$1
    local -r include_tests=$2
    find "$source_dir" -type f $include_tests -printf '%P\n' | sort
}


while getopts "b:d" opt; do
    case $opt in
        b )
            # TODO check if number
            declare -r BATCH_SIZE=$OPTARG;;
        d ) 
            declare -r DRY_RUN=;;
	esac
done
shift $(expr $OPTIND - 1 )


source_dir=${1-}
if [[ -z $(readlink -e "$source_dir") ]]; then
    echo "[ERROR] Source directory missing $source_dir"
    exit 1
fi

include_tests=${2-}
if [[ -z $include_tests ]]; then
    echo "[ERROR] Include tests for finding candidates missing" >&2
    exit 1
fi

s3_target_url=${3-}
if [[ -z $s3_target_url ]]; then
    echo "[ERROR] S3 target url missing" >&2
    exit 1
fi


# NB --old-line-format="%L" will only output the line content, no ">" suffixes
# NB wrapping the diff call in an echo call will help with handling the return code from diff:
#   if there are differences, diff will always return 1 but we just want its output
files=$(echo "$(diff --new-line-format= --unchanged-line-format= --old-line-format="%L" \
        <(list_all_local_files "$source_dir" "$include_tests") \
        <(fetch_list_of_remote_files "$s3_target_url"))")

if [[ -z $files ]]; then
    echo "[INFO] Nothing to sync" >&2
    exit
elif [[ -v BATCH_SIZE ]]; then
    echo "[INFO] Found $(wc -l <<<"$files") files to sync, selected first $BATCH_SIZE" >&2
    files=$(head -n "$BATCH_SIZE" <<<"$files")
else    
    echo "[INFO] Found $(wc -l <<<"$files") files to sync" >&2
fi

if [[ -v DRY_RUN ]]; then
    echo "$files"
    exit
fi

sync_files "$source_dir" "$s3_target_url" <<<"$files"