deduplicate

#!/bin/bash

# Invoke with ./deduplicate {1..somenumber} for number order
#  - this will guess the file names (missing will be ignored)
# or with ./deduplicate * to check them all in lexicographical order


if ! mkdir duplicates; then
	printf "unable to create a directory for storing duplicates.\n"
	return 1
fi

declare -A sum_map

for message in "$@"; do
	if [[ ! -f "$message" ]]; then
		continue
	fi

	sum="$(sha512sum -b "$message")"
	sum="${sum%% *}" # remove everything but the sum

	if [[ -z ${sum_map[$sum]} ]]; then
		printf "inserting %s -> %s\n" "$sum" "$message"
		sum_map[$sum]="$message"

	else
		printf "dedup %s -> %s\n => from: %s\n" "$sum" "$message" "${sum_map[$sum]}"
		mv "$message" duplicates
	fi

done

if [[ duplicates/* == "duplicates/*" ]]; then
	printf "There were no duplicates.\n"
	rmdir duplicates
else
	num=(duplicates*)
	printf "There are %s duplicates in duplicates/.\n" "${#num[@]}"
fi